Ë
    L²jtv ã            -       ó¤ƒ  — d dl Z d dlmZmZ d dlmZ d dlmZ d dlm	Z	 d dl
mZ d dlZd dlmZ d dlmZmZmZ d dlmZmZmZmZ d d	lmZ d d
lmZmZ d dlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) d dl*m+Z+m,Z,m-Z-m.Z.m/Z/ d dl0m1Z1m2Z2 d dl3m4Z5 d dl6m7Z7m8Z8 d dl9m:Z;  e	d«      Z< ed«      Z=ej|                  j~                  Z?ej€                  jƒ                  ddd«      ZB eCd«      \  ZDZEZFd„ ZGd„ ZHdeee=e<f   gee=e<f   f   fd„ZIde!fd„ZJd„ ZKd„ ZL eIe?jš                  e?jœ                  g«       e/«       dddejž                  ddfd „«       «       ZP eIe?j¢                  j¤                  e?j¢                  j¦                  g«       e/«       d!„ «       «       ZT eIe?jª                  j¤                  e?jª                  j¦                  g«       e/«       d"d#œd$„«       «       ZU eIe?j¬                  «       e/«       d%„ «       «       ZV eIe?j®                  j¤                  e?j®                  j¦                  e?j°                  j¤                  e?j°                  j¦                  g«       e/d&d'«      d(„ «       «       ZY eIe?j´                  j¤                  e?j´                  j¦                  g«       e/«       d)„ «       «       ZZd*„ Z[�dLd+ed,e\e]   d-e^fd.„Z_ eIe?jÀ                  j¤                  e?jÀ                  j¦                  g«       e/«       d/„ «       «       ZadZbd,e\e]   fd0„Zc eIe?jÈ                  j¤                  e?jÈ                  j¦                  g«       e/«       d1„ «       «       Ze eIe?jÌ                  jÎ                  «      dd2œd3„«       Zh eIe?jÌ                  j¤                  «      ejÒ                  dddd4œd5„«       Zj eIe?jÖ                  j¤                  e?jÖ                  j¦                  g«       e/«       ejÒ                  dddd4œd6„«       «       Zl eIe?jÖ                  jÚ                  e?jÖ                  jÜ                  g«       e/«       ejÒ                  dddd4œd7„«       «       Zo eIe?jà                  j¤                  e?jà                  j¦                  g«       e/«       ddddd4œd8„«       «       Zq eIe?jä                  j¤                  «      d9„ «       Zs eIe?jè                  j¤                  «      d:„ «       Zud;„ Zv eIe?jî                  j¤                  «      �dMd<„«       Zx eIe?jò                  j¤                  «      �dMd=„«       Zz eIe?jö                  j¤                  e?jö                  j¦                  g«       e/«       d+ed,e\e]   d>e]d?e]fd@„«       «       Z| eIe?jú                  j¤                  «      �dLdA„«       Z~dB„ Z eIe?�j                   j¤                  «      dC„ «       Z� eIe?�j                  «      	 	 	 �dNdDedEedFedGedz  dHeƒdz  dIe�j                  dz  fdJ„«       Z… eIe?�j                  «      	 �dOdKedLedMedIe�j                  dz  fdN„«       Z‡ eIe?�j                  «      dOdOddPœdDedKedLedMedIe�j                  dz  f
dQ„«       Z‰ eIe?�j                  «      	 	 	 	 	 	 	 �dPdRej$                  dSej$                  dGedz  dTedz  dIe�j                  dz  dUe^dVe]dWe]dXe]fdY„«       Z‹ eIe?�j                  j¤                  «      dZd[œd+ed,e]d\ed]ej$                  d^eƒd_e^defd`„«       Z� eIe?�j                  j¤                  «      dZd[œd+ed,e]d\ed]ej$                  d^eƒd_e^defda„«       Z� e/«        eIe?�j                   j¤                  «      db„ «       «       Z‘ eIe?�j$                  j¤                  «      dddd dddcœdded^eƒdeedz  d'edz  dfedz  dge]dhe^defdi„«       Z“ eIe?�j(                  j¤                  e?�j(                  �j*                  g«       e/«       dj„ «       «       Z– eIe?�j(                  �j.                  «      �dLdk„«       Z˜ eIe?�j2                  j¤                  e?�j2                  �j*                  g«       e/«       dl„ «       «       Zš eIe?�j2                  �j.                  «      �dLdm„«       Z› eIe?�j8                  j¤                  «      dn„ «       Z� eIe?�j8                  j¦                  «      do„ «       Zž eIe?�j>                  j¤                  «      dp„ «       Z  eIe?�j>                  �jB                  «      dq„ «       Z¢ eIe?�jF                  j¤                  «      dr„ «       Z¤ eIe?�jJ                  j¤                  «      ddddddsœdt„«       Z¦ eIe?�jN                  j¤                  «      �dQdu„«       Z§ eIe?�jP                  j¤                  «      �dNdv„«       Z© eIe?�jT                  j¤                  «      �dQdw„«       Zª eIe?�jV                  j¤                  «      dx„ «       Z¬ eIe?�jZ                  �jB                  «      dy„ «       Z®d+edzeƒfd{„Z¯d+ed|ed}eƒfd~„Z°	 �dRdedzeƒd€e^fd�„Z±�dSd|edzeƒd‚eƒfdƒ„Z²d|ed„ed…e^dzeƒfd†„Z³	 �dTdˆeƒd‡edDed‰eƒfdŠ„Z´d‹eƒfdŒ„Zµ eIe?�jl                  j¤                  e?�jl                  �jn                  g«       e/d�dŽ«      �dUd|ed‹eƒd�e^fd‘„«       «       Z¸ eIe?�jr                  j¤                  e?�jt                  j¦                  g«       e/«       dDedefd’„«       «       Z» eIe?�jx                  g«       e/d�dŽ«      dDefd“„«       «       Z½d”edefd•„Z¾ eIe?�j~                  «       e/«       d+ed|ed–e^defd—„«       «       Z¿ eIe?�j€                  «       e/«       �dLd+ed|ed–e^defd˜„«       «       ZÀ eIe?�j‚                  «       e/«       �dLd+ed–e^defd™„«       «       ZÁ eIe?�j„                  «       e/«       �dLd+ed–e^defdš„«       «       ZÂ eIe?�j†                  j¤                  «      �dVd|ed–e^d›e^fdœ„«       ZÃ eIe?�jˆ                  j¤                  e?�jˆ                  j¦                  g«       e/«       dDed�edefdž„«       «       ZÄ eIe?�jŠ                  j¤                  «      �dLd|ed›e^fdŸ„«       ZÆ eIe?�jŽ                  j¤                  e?�jŽ                  j¦                  g«       e/d d¡d¢«      ddd£œd+ed¤e^d›e^deÈeeef   fd¥„«       «       ZÉ eIe?�j”                  j¤                  e?�j”                  j¦                  g«       e/«       dd¦œd ed¡ed„ed¤e^def
d§„«       «       ZË eIe?�j˜                  j¤                  e?�j˜                  j¦                  g«       e/d¨d�d©«      dZdªœd|ed«e^deÈeeef   fd¬„«       «       ZÍ eIe?�jœ                  j¤                  e?�jœ                  j¦                  g«       e/d­d¡d¢«      dZdd®œd|ed«e^d›e^deÈeeef   fd¯„«       «       ZÏ eIe?�j                   j¤                  e?�j                   j¦                  g«       e/«       dZdd°œd­ed¡ed„ed…e^d±e^defd²„«       «       ZÑ eIe?�j¤                  «       e/d¨d�d©«      	 	 �dWd­ed¡ed³e^d´e^deÈeeef   f
dµ„«       «       ZÓd¶eƒdeÈe^e^f   fd·„ZÔ eIe?�jª                  j¤                  e?�jª                  j¦                  g«       e/d¸d¹«      �dXd|ed¶eƒdeÈeef   fdº„«       «       ZÖ eIe?�j®                  j¤                  e?�j®                  �j°                  g«       e/d»d¼d­d¡«      d|edeÈeeeef   fd½„«       «       Z× eIe?�j²                  j¤                  «      	 	 	 �dYd|ed¾e^d¿e^dÀeƒdz  fdÁ„«       ZÚdÂedÃedeÈe\e]   e\e]   f   fdÄ„ZÛdÂedÃed}eƒdz  deÈeef   fdÅ„ZÜdDedÆede^fdÇ„ZÝ eIe?�j¼                  «      dZddddddÈœd|ed„ed…e^d›e^d‡edz  d­edz  d¡edz  d¢edz  deÈeeeef   fdÉ„«       ZÞ eIe?�j¾                  j¤                  e?�j¾                  j¦                  g«      dZdddÊœd|ed„ed–e^d…e^dËe^dÌedz  defdÍ„«       Zà eIe?�jÂ                  «       e/dÎdÏdZ¬Ð«      	 	 	 �dZd+ed|ed–e^dÑe^dËe^deÈeef   fdÒ„«       «       Zâ eIe?�jÆ                  j¤                  «      dÓ„ «       Zä eIe?�jÊ                  «       e/«       	 	 �d[dDed�edÆed…e^dÑe^defdÔ„«       «       ZådÕ„ ZædÖ„ Zç eIe?�jÐ                  «       e/«       d×„ «       «       Zé eIe?�jÔ                  «       e/«       dØ„ «       «       ZëdÙ„ Zì eIe?�jÚ                  «       e/dÚ«      dÛ„ «       «       Zî eIe?�jÞ                  «       e/dÚ«      dÜ„ «       «       ZðdÝ„ Zñ eIe?�jä                  «       e/«       dÞ„ «       «       Zó eIe?�jè                  «       e/«       dß„ «       «       Zõ eIe?�jì                  j¤                  «      dà„ «       Z÷ eIe?�jð                  j¤                  e?�jð                  �jò                  e?�jô                  j¤                  e?�jô                  �jò                  g«       e/dÚ«      dá„ «       «       Zûdâ„ Zü eIe?�jú                  «       e/«       dã„ «       «       Zþ eIe?�jþ                  «       e/«       dä„ «       «       �Z  eIe?�j                  j¤                  e?�j                  �jò                  e?�j                  j¤                  e?�j                  �jò                  g«       e/dÚ«      då„ «       «       �Z eIe?�j                  «       e/«       �d\d+edæ�edefdç„«       «       �Z eIe?�j                  «       e/«       dèed+edæ�edéedef
dê„«       «       �Z eIe?�j                  j¤                  e?�j                  j¦                  g«       e/dZ¬Ð«      dOdOdëœdì„«       «       �Z
 eIe?�j                  j¤                  e?�j                  j¦                  g«       e/«       dd2œdí„«       «       �Z eIe?�j                  �j
                  «      �d]dî„«       �Z eIe?�j                  �j                  «      �d]dï„«       �Z eIe?�j"                  j¤                  e?�j"                  j¦                  g«       e/«       �dOdð„«       «       �Z eIe?�j&                  j¤                  «      	 	 �dVdñ„«       �Z eIe?�j*                  «       e/dZ¬Ð«      �dOdIe�j                  dz  fdò„«       «       �Zdó„ �Z�d^dô„�Z	 �dOdõej$                  dEej$                  döe\e]   e]z  d÷e\e]   e]z  døe\e]   e]z  dùe^dúe]dûe\e]   e]z  dz  fdü„�Zdý„ �Z eIe?�j6                  j¤                  «      dõej$                  dEej$                  dGej$                  dz  dþej$                  dz  dÿej$                  dz  �d e^�d�e�d�ef�d„«       �Z eIe?�j:                  j¤                  «      dõej$                  dEej$                  dGej$                  döe\e]   d÷e\e]   døe\e]   dùe^dûe\e]   dúe]f�d„«       �Ze�j>                  �j@                  �rej€                  jƒ                  �ddd«      �Z! eIej|                  �jD                  �jF                  j¤                  «      �d„ «       �Z$ eIej|                  �jD                  �jJ                  j¤                  «      �d„ «       �Z&e�j>                  �jN                  rOej€                  jƒ                  �ddd«      �Z( eIej|                  �jR                  �jT                  «      �d	„ «       �Z+ej€                  jƒ                  �d
dd«      �Z, eIej|                  �jZ                  �j\                  j¤                  «       eIej|                  �jZ                  �j^                  j¤                  «       eIej|                  �jZ                  �j^                  �j`                  «      �d„ «       «       «       �Z1 eIej|                  �jZ                  �j\                  �jd                  «       eIej|                  �jZ                  �j\                  �jf                  «      �d„ «       «       �Z4 eIej|                  �jZ                  �jj                  j¤                  «       eIej|                  �jZ                  �jj                  �j`                  «      �d„ «       «       �Z6 eIej|                  �jZ                  �jj                  �jd                  «       eIej|                  �jZ                  �jj                  �jf                  «      �d„ «       «       �Z7 eIej|                  �jZ                  �jp                  j¤                  «       eIej|                  �jZ                  �jr                  j¤                  «      �d„ «       «       �Z:ej€                  jƒ                  �ddd«      �Z; eIej|                  �jx                  �jz                  «      	 	 	 	 �d_�d„«       �Z> eIej|                  �jx                  �j~                  «      �d„ «       �Z@�d„ �ZA eIe?�j„                  «      dDej$                  �d�e�de]�de�j                  dej$                  f
�d„«       �ZC eIe?�jˆ                  j¤                  «      	 	 	 	 	 �d`�d„«       �ZE�d„ �ZF eIe?�jŽ                  j¤                  «      �d„ «       �ZH eIe?�j’                  «       e/«       	 	 	 	 	 �d`�d„«       «       �ZJ eIe?�j–                  «       e/dÚ«      �d„ «       «       �ZL eIe?�jš                  j¤                  «      �d„ «       �ZN eIe?�jž                  j¤                  «      �d „ «       �ZP eIe?�j¢                  j¤                  «      �d!„ «       �ZR eIe?�j¦                  «       e/dÚ«      �d"„ «       «       �ZT�d#ed‚eƒf�d$„�ZU eIe?�j¬                  «       e/dÌd'«      �d%„ «       «       �ZW eIe?�j°                  «       e/dÚ«      �d&„ «       «       �ZY eIe?�j´                  «       e/dÌd'«      �d'„ «       «       �Z[ eIe?�j¸                  «       e/dÚ«      �d(„ «       «       �Z] eIe?�j¼                  j$                  «      �dO�d)„«       �Z_ eIe?�jÀ                  j¤                  e?�jÀ                  j¦                  g«       e/«       �d*„ «       «       �Za eIe?�jÄ                  j¤                  e?�jÄ                  j¦                  g«       e/«       d"�d+œ�d,e]f�d-„«       «       �Zb eIej|                  j~                  �jÆ                  j¤                  ej|                  j~                  �jÆ                  j¦                  g«       e/«       �d.„ «       «       �Zc eIe?�jÈ                  j$                  e?�jÊ                  j$                  g«      �d/„ «       �Zf eIe?�jÎ                  j¤                  g«      �d0„ «       �Zh eIe?�jÒ                  j¤                  e?�jÒ                  j¦                  g«       e/dZ¬Ð«      dOdOdëœ�d1„«       «       �Zj eIe?�jÖ                  j$                  g«      �d2„ «       �Zl eIe?�jÚ                  j¤                  e?�jÜ                  j¤                  g«      dd�d3œ�d4„«       �Zo eIe?�jà                  j¤                  g«      dd�d3œ�d5„«       �Zq eIe?�jä                  g«       e/«       �d6„ «       «       �Zs eIe?�jè                  g«      �d7„ «       �Zu eIe?�jì                  g«      �d8„ «       �Zw eIe?�jð                  g«      �d9„ «       �Zy eIe?�jô                  g«      �d:„ «       �Z{ eIe?�jø                  g«      �d;„ «       �Z|�d<e]�d=e]de]f�d>„�Z}�d?„ �Z~ eIe?�jþ                  g«      dGedz  f�d@„«       �Z€ eIe?�j                  g«      �dA„ «       �Z‚ eIe?�j                  g«      �dB„ «       �Z„ eIe?�j
                  j¤                  «      �dC„ «       �Z† eIe?�j                  «       e/«       �dD„ «       «       �Zˆ eIe?�j                  j¤                  «      	 	 	 	 	 	 �da�dE„«       �ZŠ eIe?�j                  j¤                  «      �dF„ «       �ZŒ�dR�dG„�Z� eIe?�j                  j¤                  e?�j                  j¦                  g«       e/«       �dbd�dHœ�dI„«       «       �Z� eIe?�j                   j¤                  e?�j"                  j¤                  g«      �dJ„ «       �Z’ eIe?�j                   �j.                  e?�j                   �j&                  e?�j"                  �j.                  e?�j"                  �j&                  e?�j(                  j¤                  e?�j(                  �j*                  g«       e/d&d'«      �dc�dK„«       «       �Z– eIe?�j.                  j¤                  «      �dL„ «       �Z˜ eIe?�j2                  j¤                  «      �dM„ «       �Zš eIe?�j6                  j¤                  «      �dN„ «       �Zœ eIe?�j:                  �j<                  e?�j>                  �j<                  e?�j:                  j$                  e?�j>                  j$                  e?�j@                  j¤                  e?�jB                  j¤                  e?�jD                  j¤                  g«      �dO„ «       �Z£ eIe?�jH                  �j<                  e?�jJ                  �j<                  e?�jH                  j$                  e?�jJ                  j$                  g«      �d�dP„«       �Z¦ eIe?�jN                  �j<                  e?�jP                  �j<                  g«      �d�dQ„«       �Z© eIe?�jT                  j¤                  e?�jT                  �jV                  g«      �dR„ «       �Z¬�dS„ �Z­ eIe?�j\                  j$                  e?�j\                  �j<                  g«      �dT„ «       �Z¯ eIe?�j`                  j$                  e?�j`                  �j<                  g«      �dU„ «       �Z± eIe?�jd                  j¤                  «      �dV„ «       �Z³ eIe?�jh                  j$                  e?�jh                  �j<                  g«      �dW„ «       �Zµ eIe?�jl                  j$                  e?�jl                  �j<                  g«      �dX„ «       �Z· eIe?�jp                  j¤                  «      �dY„ «       �Z¹ eIe?�jt                  j$                  «       e/«       �ddef�dZ„«       «       �Z» eIe?�jx                  g«       e/«       	 �dd�d[„«       «       �Z½ eIe?�j|                  g«      	 �dd�d\„«       �Z¿ eIe?�j€                  g«      	 �dd�d]„«       �ZÁ eIe?�j„                  j¤                  e?�j†                  j¤                  g«      �dL�d^„«       �ZÄ eIe?�jŠ                  �j<                  «      �d_„ «       �ZÆ eIe?�jŽ                  j¤                  «      �d`„ «       �ZÈ eIe?�j’                  «      �da„ «       �ZÊ eIe?�j–                  «       e/«       �db„ «       «       �ZÌ eIe?�jš                  «      �dc„ «       �ZÎ eIe?�jž                  j¤                  «      �dL�dd„«       �ZÐ�dQ�de„�ZÑ eIe?�j¤                  j¤                  e?�j¤                  j¦                  g«       e/dZ¬Ð«      �df„ «       «       �ZÓ eIe?�j¤                  �j                  e?�j¤                  �j¨                  g«       e/dZ¬Ð«      �dg„ «       «       �ZÕ�dh„ �ZÖ�di„ �Z×�dj„ �ZØ�dk„ �ZÙ	 �dLdDe�dle]�dme]�dne]�doe]�dpe]�dqe]�dre]�dse]�dte]�due]�dve]�dwe]�dxe]�dye]�dze]�d{e]�d|e]�d}e]�d~e]dˆeƒ�de^f,�d€„�ZÚ�d�„ �ZÛdDe�d#e�dle]�dme]�dne]�doe]�dpe]�dqe]�dre]�dse]�dte]�due]�dye]�dze]�d{e]�d|e]�d}e]�d~e]dˆeƒf&�d‚„�ZÜ�dƒ„ �ZÝ eIe?�j¼                  «       e/dÚ«      �d„„ «       «       �Zß eIe?�jÀ                  j¤                  «      	 	 	 	 �d_�d…„«       �Zá eIe?�jÄ                  j¤                  «      �d†„ «       �Zã eIe?�jÈ                  «       e/dÌd'«      	 	 	 	 �d_�d‡„«       «       �Zå eIe?�jÌ                  «       e/dÚ«      �dˆ„ «       «       �ZçdDe�d‰ef�dŠ„�Zè G �d‹„ �dŒe«      �ZédDe�d‰e�d�e]f�dŽ„�Zê eIe?�jÖ                  j¤                  «      �d�„ «       �Zì eIe?�jÚ                  «       e/«       �d�„ «       «       �Zí eIe?�jÜ                  «       e/dÚ�d‘«      �d’„ «       «       �Zî eIe?�jÞ                  j¤                  g«      �d“„ «       �Zï eIe?�jà                  j¤                  «      	 	 	 	 	 �de�d”„«       �Zð eIe?�jâ                  j¤                  e?�jâ                  j¦                  g«       e/«       ddddd�d•œ�d–„«       «       �Zò eIe?�jæ                  j¤                  e?�jæ                  j¦                  g«       e/«       ddddd�d•œ�d—„«       «       �Zô eIe?�jê                  j¤                  «      �d˜„ «       �Zö eIe?�jî                  j¤                  «      �df�d™„«       �Zø�dš„ �Zù�dRd,e]�d›e]�dœe^f�d�„�Zú�dž„ �Zû�dŸ„ �Zü eIe?�jú                  j¤                  «      �dL�d „«       �Zþ�dL�d¡„�Zÿ�dO�d¢„�Z �d£„ �Z�dO�d¤„�Z�dg�d¥„�Z eIe?�j                  j¤                  «      �d¦„ «       �Z eIe?�j                  «      �d§„ «       �Z eIe?�j                  �j                  e?�j                  �j                  e?�j                  �j                  e?�j                  �j                  g«       e/«       �dO�d¨„«       «       �Z eIe?�j                  �j                  e?�j                  �j                  e?�j                  �j                  e?�j                  �j                  g«      �dO�d©„«       �Z eIe?�j                   j¤                  g«      	 	 	 	 �dh�dªe�d«e�d¬e�d­�e�d®e^�d¯e^�d�edz  f�d°„«       �Z eIe?�j                   �jx                  g«      	 	 	 	 �dh�dªe�d«e�d¬e�d±edz  �d²edz  �d³edz  �d­�e�d®e^�d¯e^�d�edz  f�d´„«       �Z�dªe�dµeÈe]�d¶f   f�d·„�Z eIe?�j(                  g«      	 	 	 	 �dh�dªe�d«e�d¬e�d¸edz  �d¹e^�d­�e�d®e^�d¯e^�d�edz  f�dº„«       �Z eIe?�j,                  g«      	 	 	 	 	 �di�dªe�d«e�d¬e�d¸edz  �d­�e�d®e^�d¯e^�d�edz  f�d»„«       �Z eIe?�j0                  «      d�d¼œ�d½e�dªe�d«e�d¬e�d¸e�d¾e\e^   dÌe�d¿e�dÀe�dÁe�dÂe]�dÃe]�d­�e�d®e^�dÄe�dÅe�d�edz  f"�dÆ„«       �Z eIe?�j4                  g«      	 �dO�d½e�dªe�d«e�d¬edÌe�d¿e�dÀe�dÁe�dÂe]�dÃe]�d­�e�d®e^�dÄe�dÅe�d�edz  f�dÇ„«       �Z eIe?�j8                  g«      	 	 	 	 �dj�dªe�d«e�d¬e�d­�e�d®e^�dÈedz  �d�edz  f�dÉ„«       �Z eIe?�j<                  g«      	 	 �dQ�d½e�dªe�d«e�d¬edÌe�d¿e�d­�e�d®e^�dÈedz  �d�edz  f�dÊ„«       �Z eIe?�j@                  g«      	 	 	 	 	 	 �dk�dªe�d«e�d¬e�dÈedz  �d­�e�d®e^�dËedz  �d�edz  �dÌe^deÈeef   f�dÍ„«       �Z! eIe?�jD                  g«      	 	 	 �dl�dªe�d«e�d¬e�d¸edz  �d¹e^�d®e^�d�edz  f�dÎ„«       �Z# eIe?�jH                  g«      	 	 �dm�d½e�dªe�d«e�d¬e�d¸edz  dÌe�d¿e�dÄe�dÅe�d­�e�d¾e\e^   �d®e^�d�edz  f�dÏ„«       �Z% eIe?�jL                  g«      	 �dO�d½e�dªe�d«e�d¬edÌe�d¿e�dÄe�dÅe�d¸e�dÀe�dÁe�dÂe]�dÃe]�d­�e�d®e^�d�edz  f �dÐ„«       �Z' eIe?�jP                  j¤                  g«      	 	 	 	 	 	 	 �dn�dªe�d«e�d¬e�dÀedz  �dÁedz  �dÂe]�dÃe]�d­�e�d®e^�d¯e^�d�edz  �dÑe]dz  �dÒe]dz  �dÓedz  �dÔedz  �dÕedz  �dÖe]dz  f"�d×„«       �Z) eIe?�jT                  j¤                  g«      	 	 	 	 	 	 	 �dndÌe�dªe�d«e�d¬e�dÀedz  �dÁedz  �dÂe]�dÃe]�d­�e�d®e^�d¯e^�d�edz  �dÑe]dz  �dÒe]dz  �dÓedz  �dÔedz  �dÕedz  �dÖe]dz  f$�dØ„«       �Z+ eIe?�jP                  �jx                  g«      	 	 	 	 	 �de�dªe�d«e�d¬e�dÀedz  �dÁedz  �dÂe]�dÃe]�d­�e�d®e^�d¯e^�d±edz  �d²edz  �d³edz  �d�edz  �dÑe]dz  �dÒe]dz  �dÓedz  �dÔedz  f$�dÙ„«       �Z, eIe?�jZ                  g«      	 	 	 �dN�d½e�dªe�d«e�d¬edÌe�d¿e�dÀe�dÁe�dÂe]�dÃe]�d­�e�d®e^�dÄe�dÅe�d�edz  �dÑe]dz  �dÒe]dz  f"�dÚ„«       �Z. eIe?�j^                  g«      	 	 	 	 	 �do�dªe�d«e�d¬edGedz  �dÛedz  �dÜedz  �dÝe]dz  �dÞe]dz  �d­�e�dße]�d¹e^�d�edz  �dàedz  �dáedz  �dâe]dz  f�dã„«       �Z0 eIe?�jb                  g«      	 	 	 �dg�d½e�dªe�d«e�d¬edGedz  �dÛedz  �dÜedz  �dÝe�jd                  �dÞe�jd                  �d¿e�d­�e�dÄe�dÅe�dße]�däe^�d�edz  �dåe]dz  �dæe^f$�dç„«       �Z3	 	 	 	 �dpd+ej$                  dMej$                  �dèej$                  �déej$                  dGej$                  dz  �dêej$                  dz  dIe�j                  dz  �dëe^f�dì„�Z4 eIe?�jj                  j¤                  g«      	 	 	 	 �dpd+ej$                  dMej$                  �dèej$                  �déej$                  dGej$                  dz  �dêej$                  dz  dIe�j                  dz  �dëe^f�dí„«       �Z6	 	 	 	 	 �dqd+ej$                  dMej$                  �dèe\ej$                     �dîe\e7   �dée\ej$                     �dïe\e7   dGej$                  dz  dIe�j                  dz  �dðe\e8   dz  �dñe\e8   dz  �dëe^f�dò„�Z7 eIe?�jp                  j¤                  g«      	 	 	 	 �dpd+ej$                  dMej$                  �dèe\ej$                     �dîe\e7   �dðe\e8   �dée\ej$                     �dïe\e7   �dñe\e8   dGej$                  dz  �dóe�j                  dz  �dôe\e]   dz  �dëe^f�dõ„«       �Z9 eIe?�jt                  �jv                  e?�jt                  �jx                  g«       e/«       �dR�dö„«       «       �Z= eIe?�j|                  �jv                  «      �dR�d÷„«       �Z? eIe?�j€                  j¤                  e?�j€                  j¦                  g«       e/«       �dLdd2œ�dø„«       «       �ZA�dù„ �ZB�dú„ �ZC eIe?�jˆ                  j¤                  e?�jŠ                  j¤                  g«      �dO�dû„«       �ZD eIe?�jŒ                  j¤                  e?�jŽ                  j¤                  g«      �dQ�dü„«       �ZF eIe?�j�                  j¤                  e?�j’                  j¤                  g«      	 	 �dQ�d#e�dýee]e�jd                  z     �dþee]e�jd                  z     �dÿ�edz  �d �edz  f
�d„«       �ZH eIe?�j”                  j¤                  e?�j–                  j¤                  g«      �dN�d„«       �ZJ eIe?�j˜                  j¤                  e?�j˜                  �jš                  e?�j˜                  �j*                  e?�j˜                  �jœ                  g«      �dr�d„«       �ZO�d„ �ZP eIe?�j¢                  j¤                  «      	 	 �dQ�d„«       �ZR eIe?�j¦                  j¤                  «      �d„ «       �ZS eIe?�j¨                  j¤                  «      �d„ «       �ZT eIe?�jª                  j¤                  «      �d„ «       �ZU�d	„ �ZV�d
„ �ZW eIe?�j°                  j¤                  e?�j²                  j¤                  g«      �db�d„«       �ZZ eIe?�j¶                  j¤                  «      �ds�d„«       �Z[ eIe?�j¸                  j¤                  «      �dt�d„«       �Z] eIe?�j¼                  «       e/«       	 �du�d„«       «       �Z_ eIe?�jÀ                  j¤                  e?�jÀ                  �j*                  g«       e/d&d'«      �dc�d„«       «       �Zae�jÄ                  �Zc�d„ �Zd eIe?�jÊ                  j¤                  «      �d„ «       �Ze eIe?�jÌ                  j¤                  «      �d„ «       �Zf eIe?�jÎ                  j¤                  «      �d„ «       �Zh eIe?�jÒ                  j¤                  «      �d„ «       �Zi eIe?�jÔ                  j$                  e?�jÔ                  �jÖ                  g«       e/«       dd�dœ�d„«       «       �Zl eIe?�jÔ                  �j<                  e?�jÔ                  �jÚ                  g«      dd�dœd+e&�de�de^�de^f�d„«       �Zn eIe?�jÞ                  g«       e/«       �dv�d„«       «       �Zp eIe?�jâ                  j¤                  e?�jä                  j¤                  e?�jæ                  j¤                  g«      	 	 �dQ�d„«       �Zt eIe?�jê                  j¤                  e?�jì                  j¤                  g«      	 	 �dQ�d„«       �Zw eIe?�jð                  j¤                  «      �d„ «       �Zx eIe?�jò                  j¤                  e?�jò                  j¦                  g«       e/«       �dN�d„«       «       �Zy eIej|                  j~                  �jô                  «      �d „ «       �Zz eIej|                  j~                  �jö                  «      �d!„ «       �Z{ eIe?�jø                  «       e/«       dddd�d"œ�d#„«       «       �Z}�d$„ �Z~ eIe?�jþ                  «      �d%„ «       �Z€ eIe?�j                  «      	 �dw�d&„«       �Z‚ eIe?�j                  «      	 �dw�d'„«       �Z„ eIe?�j
                  «      	 �dw�d(„«       �Z† eIe?�j                  «       e/«       dd�d)œ�d*„«       «       �Zˆ eIe?�j                  «       e/«       �d+e]d+edef�d,„«       «       �ZŠ eIe?�j                  «      d+ef�d-„«       �ZŒ eIe?�j                  «       e/dZ¬Ð«      d+edef�d.„«       «       �Z� eIe?�j                  «       e/«       d+edef�d/„«       «       �ZŽ�d0„ �Z�	 	 	 	 	 �dq�d1e�d2e�dèej$                  dz  �déej$                  dz  �d3edz  dGedz  �dêej$                  dz  dIe�j                  dz  �dëe^f�d4„�Z� eIe?�j"                  «       e/«       	 	 	 �dN�d1e�d2e�d3edz  dGedz  dIe�j                  dz  def�d5„«       «       �Z’ eIe?�j&                  g«      	 	 	 	 	 �dq�d1ej$                  �d2ej$                  �dèej$                  �déej$                  �d3ej$                  dz  dGej$                  dz  �dêej$                  dz  dIe�j                  dz  �dëe^f�d6„«       �Z” eIe?�j*                  �j<                  «      �dx�d7„«       �Z– eIe?�j.                  «       e/«       �d8ed,e]�d9e^def�d:„«       «       �Z˜ eIe?�j2                  «       e/«       �d�d;„«       «       �Zš eIe?�j6                  «       e/«       	 	 	 �dydEed'e�d<e]�d=e^�d>e^def�d?„«       «       �Z› eIe?�j8                  j¤                  «      	 �dzd&edfe\e   �d@e\e]   �dA�ef�dB„«       �Z��dC„ �Zž eIe?�j>                  j¤                  «      	 	 	 	 �d{�dD„«       �Z �dE„ �Z¡ �eže?�jD                  «        �eže?�jF                  «        �eže?�jH                  «        �eže?�jJ                  «        �eže?�jL                  «        �eže?�jN                  «        �eže?�jP                  «        �eže?�jR                  «        �eže?�jT                  «        �e¡e?�jV                  «        �e¡e?�jX                  «        �e¡e?�jZ                  «        �e¡e?�j\                  «        �e¡e?�j^                  «        �e¡e?�j`                  «        �e¡e?�jb                  «        �e¡e?�jd                  «        �e¡e?�jf                  «        �e¡e?�jh                  «        �e¡e?�jj                  «        �e¡e?�jl                  «       �dF„ �Z· eIe?�jp                  «       e/«       �dG„ «       «       �Z¸ eIe?�jr                  «       e/«       dO�dHœ�dI„«       «       �Z¹ eIe?�jt                  «       e/«       dO�dHœ�dJ„«       «       �Zº �e·e?�jp                  «      �Z» �e·e?�jr                  «      �Z¼ �e·e?�jt                  «      �Z½d dl0Zd d�l¾Zd d�l¿Z�dK„ �ZÀ �eÀ«        y(|  é    N)ÚCallableÚSequence)ÚEnum)Úwraps)ÚTypeVar)Ú	ParamSpec)ÚSymBoolÚSymFloatÚTensor)Ú_add_op_to_registryÚ_convert_out_paramsÚglobal_decomposition_tableÚ
meta_table)Ú
OpOverload)Ú_prim_elementwise_metaÚ$ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND)ÚBoolLikeÚcorresponding_complex_dtypeÚcorresponding_real_dtypeÚelementwise_dtypesÚELEMENTWISE_TYPE_PROMOTION_KINDÚ	FloatLikeÚIntLikeÚmake_contiguous_strides_forÚNumberÚ
NumberTypeÚsuggest_memory_formatÚsym_minÚ
TensorLike)Ú_maybe_convert_to_dtypeÚ_maybe_resize_outÚ_resize_output_checkÚ_safe_copy_outÚout_wrapper)Ú_broadcast_shapesÚ_maybe_broadcast)Ú_config)ÚScalingTypeÚSwizzleType)Ú_pytreeÚ_TÚ_PÚatenÚIMPLÚMetaé   c                 ó   — | |z   dz
  |z  S ©Né   © ©ÚaÚbs     úX/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/torch/_meta_registrations.pyÚceil_divr9   9   s   € Ø�‰E�A‰I˜!ÑÐó    c                 ó   — | |z   dz
  |z  |z  S )z$Rounds up x to nearest multiple of yr3   r4   ©ÚxÚys     r8   Úround_upr?   =   s   € à�‰U�Q‰Y˜1Ñ Ñ!Ð!r:   Úreturnc                 ó   ‡ — ˆ fd„}|S )Nc                 óV   •‡ — t        ‰ «      Š ˆ fd„}t        j                  |‰«       ‰ S )Nc                 ó(   •— t        t        | ‰«       y ©N)r   r   )ÚopÚfns    €r8   Úregisterz0register_meta.<locals>.wrapper.<locals>.registerF   s   ø€ Ü¤
¨B°Õ3r:   )r   ÚpytreeÚ	tree_map_)rF   rG   rE   s   ` €r8   Úwrapperzregister_meta.<locals>.wrapperC   s)   ù€ Ü  Ó$ˆô	4ô 	×Ñ˜ 2Ô&Øˆ	r:   r4   )rE   rJ   s   ` r8   Úregister_metarK   B   s   ø€ ôð €Nr:   Útype_promotionc                 ó°   — t        j                  |d| iŽ\  }}|D �cg c]  }t        ||«      ‘Œ }}t        |Ž }t	        |dt
        j                  iŽS c c}w )NÚtype_promotion_kindrL   )Úutilsr   r    r&   r   r   ÚDEFAULT)rL   ÚargsÚ_Úresult_dtyper=   s        r8   Úelementwise_metarT   O   ss   € ô
 ×.Ñ.Ø	ðà*ñ�O€A€|ð ?CÖC¸Ô# A |Õ4ÐC€DÐCô ˜TÐ"€Dô "Ø	ðÜB×JÑJñð ùò Ds   �Ac                 óÞ   — t         j                  t         j                  t         j                  t         j                  t         j
                  t         j                  i}|j                  | | «      S rD   )ÚtorchÚ	complex32ÚhalfÚcfloatÚfloatÚcdoubleÚdoubleÚget)ÚdtypeÚfrom_complexs     r8   ÚtoRealValueTyper`   c   sE   € ä�‰œŸ™Ü�‰”e—k‘kÜ�‰”u—|‘|ð€Lð
 ×Ñ˜E 5Ó)Ð)r:   c                 ól   ‡ ‡— t        t        ‰ g|¢­Ž «      Št        j                  ‰‰ k(  ˆˆ fd„«       y )Nc                  ó   •— d‰› d‰ › �S )Nzoutput with shape z# doesn't match the broadcast shape r4   )Úbroadcasted_shapeÚ
self_shapes   €€r8   ú<lambda>z)check_inplace_broadcast.<locals>.<lambda>p   s   ø€ Ð$ Z LÐ0SÐTeÐSfÐg€ r:   )Útupler%   rV   Ú_check)rd   Ú
args_shaperc   s   ` @r8   Úcheck_inplace_broadcastri   l   s0   ù€ ÜÔ/°
ÐH¸ZÒHÓIÐÜ	‡L�LØ˜ZÑ'Ügõr:   Fc	                 ó�  ‡ ‡‡‡‡	— t        ‰ t        j                  «      r(t        j                  ‰ j	                  «       dk(  d„ «       t        ‰t        j                  «      r(t        j                  ‰j	                  «       dk(  d„ «       t        d„ ‰ ‰‰fD «       «      rZt        j                  t        j                  «       «      Š	‰€‰	ŠnFt        j                  t        j                  ‰«      ˆ	ˆfd„«       n‰xs t        j                  «       Št        ‰t        j                  «      st        dt        ‰«      › �«      ‚t        j                  t        ‰t        «      ˆˆ ˆfd„«       t        ‰t        «      st        dt        ‰«      › �«      ‚t        j                  ‰dk\  d	„ «       t        j                  ‰f‰|d
||¬«      S )Nr   c                   ó   — y©Nz:linspace only supports 0-dimensional start and end tensorsr4   r4   r:   r8   re   z(meta_linspace_logspace.<locals>.<lambda>„   ó   � r:   c                   ó   — yrl   r4   r4   r:   r8   re   z(meta_linspace_logspace.<locals>.<lambda>‰   rm   r:   c              3   ó<   K  — | ]  }t        |t        «      –— Œ y ­wrD   )Ú
isinstanceÚcomplex)Ú.0Úargs     r8   ú	<genexpr>z)meta_linspace_logspace.<locals>.<genexpr>Œ   s   è ø€ Ò
C¨Œ:�cœ7×#Ñ
Cùs   ‚c                  ó   •— d‰ › d‰› �S )Nzlinspace(): inferred dtype z& can't be safely cast to passed dtype r4   )Údefault_complex_dtyper^   s   €€r8   re   z(meta_linspace_logspace.<locals>.<lambda>•   s   ø€ Ð5Ð6KÐ5LÐLrÐsxÐryÐz€ r:   zdtype must be torch.dtype, got c                  óŽ   •— dt        ‰«      j                  › dt        ‰ «      j                  › dt        ‰«      j                  › d�S )Nz4received an invalid combination of arguments - got (ú, ú))ÚtypeÚ__name__)ÚendÚstartÚstepss   €€€r8   re   z(meta_linspace_logspace.<locals>.<lambda>Ÿ   sD   ø€ ð Ü€uƒ+×ÑÐ�rœ$˜s›)×,Ñ,Ð-¨R´°U³×0DÑ0DÐ/EÀQðH€ r:   zsteps must be IntLike, got c                   ó   — y)Nz$number of steps must be non-negativer4   r4   r:   r8   re   z(meta_linspace_logspace.<locals>.<lambda>¤   rm   r:   Úmeta©r^   ÚlayoutÚdeviceÚ
pin_memoryÚrequires_grad)rp   rV   r   rg   ÚdimÚanyrO   r   Úget_default_dtypeÚis_complex_dtyper^   ÚAssertionErrorrz   Ú_check_typer   Úempty)
r}   r|   r~   Úbaser^   rƒ   r‚   r„   r…   rv   s
   ``` `    @r8   Úmeta_linspace_logspacerŽ   t   sn  ü€ ô �%œŸ™Ô&Ü�‰Ø�I‰I‹K˜1ÑÙPô	
ô �#”u—|‘|Ô$Ü�‰Ø�G‰G‹I˜‰NÙPô	
ô
 Ñ
C°°s¸EÐ/BÔ
CÔCÜ %× AÑ AÜ×#Ñ#Ó%ó!
Ðð ˆ=Ø)‰Eä�L‰LÜ×&Ñ& uÓ-Üzõð
 Ò2œ×0Ñ0Ó2ˆÜ�eœUŸ[™[Ô)ÜÐ>¼tÀE»{¸mÐLÓMÐMô 
×ÑÜ�5œ'Ó"õ	Hôô
 �eœWÔ%ÜÐ:¼4À»;¸-ÐHÓIÐIÜ	‡L�L�˜!‘ÑKÔLä�;‰;Ø	ˆØØØØØ#ôð r:   c                 ó  ‡— t        j                  ‰j                  t         j                  k(  ˆfd„«       t        j                  | j                  «       dk(  xr ‰j                  «       dk7   d„ «       | j                  ‰j                  «      S )Nc                  ó"   •— d‰ j                   › �S )Nz2take(): Expected a long tensor for index, but got ©r^   ©Úindexs   €r8   re   zmeta_take.<locals>.<lambda>¶   s   ø€ ÐDÀUÇ[Á[ÀMÐR€ r:   r   c                   ó   — y)Nz*take(): tried to take from an empty tensorr4   r4   r:   r8   re   zmeta_take.<locals>.<lambda>»   rm   r:   )rV   rg   r^   ÚlongÚ_check_indexÚnumelÚ	new_emptyÚshape)Úselfr“   s    `r8   Ú	meta_taker›   °   sm   ø€ ô 
‡L�LØ�‰”u—z‘zÑ!ÛRôô
 
×ÑØ�Z‰Z‹\˜QÑÒ5 5§;¡;£=°AÑ#5Ð6Ù<ôð �>‰>˜%Ÿ+™+Ó&Ð&r:   éÿÿÿÿ©r†   c                óT  ‡ ‡‡— ‰ j                   }‰j                   }t        j                  ||k(  d„ «       t        j                  ‰ j                  ‰«      dk(  xr ‰j                  ‰«      dk(  ˆˆˆ fd„«       t	        ‰ j
                  ‰j
                  «      }‰ j                  |«      S )Nc                   ó   — y)Nz=linalg.cross: inputs must have the same number of dimensions.r4   r4   r:   r8   re   zlinalg_cross.<locals>.<lambda>Ç   rm   r:   r0   c                  óV   •— d‰ › d‰j                  ‰ «      › d‰j                  ‰ «      › �S )Nzlinalg.cross: inputs dimension z must have length 3. Got ú and ©Úsize)r†   Úotherrš   s   €€€r8   re   zlinalg_cross.<locals>.<lambda>Ë   s6   ø€ Ø-¨c¨Uð 3Ø—9‘9˜S“>Ð" %¨¯
©
°3«Ð'8ð:ð r:   )ÚndimrV   rg   r£   r%   r™   r˜   )rš   r¤   r†   Úx_dÚy_dÚ	out_shapes   ```   r8   Úlinalg_crossr©   À   s‡   ú€ ð �)‰)€CØ
�*‰*€CÜ	‡L�LØˆs‰
ÙOôô 
‡L�LØ�	‰	�#‹˜!ÑÒ4 §
¡
¨3£°1Ñ 4õ	
ôô " $§*¡*¨e¯k©kÓ:€IØ�>‰>˜)Ó$Ð$r:   c                 ó|   — t        | d«       t        | d«       t        j                  | t        j                  ¬«      S )Nzlinalg.matrix_exp©Úmemory_format)ÚsquareCheckInputsÚcheckFloatingOrComplexrV   Ú
empty_likeÚcontiguous_format©rš   s    r8   Úlinalg_matrix_expr²   Ô   s3   € ô �dÐ/Ô0Ü˜4Ð!4Ô5Ü×Ñ˜D´×0GÑ0GÔHÐHr:   ÚvaluesÚindicesc                 óZ  — t        j                  | j                  | j                  | j                  ¬«      }t        j                  | j                  | j                  t         j
                  ¬«      }| j                  «       dk7  r%| j                  dk7  rt        || j                  «       ||fS )N©rƒ   r^   r   )	rV   rŒ   r™   rƒ   r^   Úint64r—   r¥   Úmaybe_wrap_dim)rš   r†   r³   r´   s       r8   Ú	cummaxminr¹   Ü   sp   € ô
 �[‰[˜Ÿ™¨D¯K©K¸t¿z¹zÔJ€FÜ�k‰k˜$Ÿ*™*¨T¯[©[ÄÇÁÔL€GØ‡z�zƒ|�qÒ˜TŸY™Y¨!š^ä�s˜DŸI™IÔ&Ø�7ˆ?Ðr:   c                 óx   — t        || j                  «       t        j                  | t        j                  ¬«      S ©Nr«   )r¸   r¥   rV   r¯   r°   )rš   r†   s     r8   Úlogcumsumexpr¼   é   s+   € ô �3˜Ÿ	™	Ô"Ü×Ñ˜D´×0GÑ0GÔHÐHr:   c                ó$  ‡— |j                   }t        |«      }||z
  }t        t        |«      «      }t        |«      D �	cg c]  }	d‘Œ }
}	|D ]  }d|
|<   Œ	 |D �cg c]
  }|
|   rŒ	|‘Œ }}|t        |«      z   }t        |«      }|j	                  «       Š|d | }|j                  ˆfd„d¬«       |||d  z   }|j                  |«      }dgt        |j                  |d  «      z   }|j                  |«      }|j                  d«      }||d<   t        |«      }t        t        |«      «      D ]  }|||      ||dz   <   Œ | j                  |t        j                  ¬«       t        |«      D �	cg c]  }	d‘Œ }}	d}|dz
  }|dk\  r0|| j	                  d«      z  |||   <   ||||      z  }|dz  }|dk\  rŒ0t        ||«      D ]  }| j	                  d||z
  z   «      |||   <   Œ! | j                  ||| j                  «       «       | S c c}	w c c}w c c}	w )	NFTc                 ó   •— ‰|    S rD   r4   )r=   Úself_stridess    €r8   re   z_exec_fft.<locals>.<lambda>	  s   ø€ ˜<¨™?€ r:   ©ÚkeyÚreverserœ   r   r3   r«   )r¥   ÚlenÚlistÚrangeÚstrideÚsortÚpermuter™   Úreshaper£   Úresize_rV   r°   Úas_strided_Ústorage_offset)Úoutrš   Ú	out_sizesr†   Úforwardr¥   Úsignal_ndimÚ
batch_dimsÚdim_permuterR   Úis_transformed_dimÚdÚleftÚ	batch_endÚtmpÚinputÚbatched_sizesÚ
batch_sizeÚbatched_out_sizesÚiÚout_stridesÚbatch_numelr¿   s                         @r8   Ú	_exec_fftrß   ö   sG  ø€ Ø�9‰9€DÜ�c“(€KØ˜Ñ#€Jô ”u˜T“{Ó#€Kä).¨t«Ö5 Aš%Ð5ÐÐ5Øò %ˆØ $Ð˜1Òð%ð #Ö@�!Ð*<¸QÓ*?ŠAÐ@€DÐ@Øœ˜c›Ñ"€KÜ�D“	€Ià—;‘;“=€LØ
�j�yÐ
!€CØ‡H�HÓ*°D€HÔ9Ø˜ I JÐ/Ñ/€KØ�L‰L˜Ó%€Eð �Dœ4 §¡¨J¨KÐ 8Ó9Ñ9€MØ�M‰M˜-Ó(€Eà—‘˜A“€JØ!€M�!ÑÜ˜]Ó+ÐÜ”3�s“8‹_ò 5ˆØ#,¨S°©VÑ#4Ð˜!˜a™%Ò ð5à‡K�KÐ!´×1HÑ1H€KÔIô $ D›kÖ*˜’1Ð*€KÐ*Ø€KØ�Q‰€AØ
ˆqŠ&Ø&1°C·J±J¸q³MÑ&Aˆ�K ‘NÑ#Ø�y ¨Q¡Ñ0Ñ0ˆØ	ˆQ‰ˆð ˆq‹&ô �:˜tÓ$ò GˆØ&)§j¡j°°a¸*±nÑ1EÓ&Fˆ�K ‘NÒ#ðGà‡O�O�I˜{¨C×,>Ñ,>Ó,@ÔAà€JùòM 6ùò
 Aùò, +s   ¿	HÁ
HÁ%HÅ+	Hrš   r†   Úexclude_lastc                 óš   ‡— t        |«      }| j                  «       Št        |«      t        |«      z
  }t	        |d | ˆfd„d¬«      |d | |S )Nc                 ó   •— ‰|    S rD   r4   )rÜ   r¿   s    €r8   re   z_sort_dims.<locals>.<lambda>,  s   ø€ ¨°a©€ r:   TrÀ   )rÄ   rÆ   rÃ   ÚintÚsorted)rš   r†   rà   Úsorted_dimsr|   r¿   s        @r8   Ú
_sort_dimsræ   '  sS   ø€ Ü�s“)€KØ—;‘;“=€LÜ
ˆkÓ
œS Ó.Ñ
.€CÜØ�D�SÐÓ8À$ô€K��Ðð Ðr:   c                 óú  ‡
— t        j                  | j                  j                  «       |s| j	                  «       S t        | «      dk(  rGt         j                  j                  j                  «       s| j                  | j                  «       «      S | j                  «       }| j                  |«      }t        | «      dk7  rt        | |«      }t        || |||¬«      S t        |«      }| }	 |j                  «       Š
|j                  ˆ
fd„d¬«       t!        t"        t%        |«      «      }|t%        |«      |z
  d  }	t        ||||	|¬«       |d t%        |«      |z
   }|s|S || u r|}| j                  |«      }n||}}Œ’)NÚcpuÚcuda©rÏ   Tc                 ó   •— ‰|    S rD   r4   ©rÜ   Ústridess    €r8   re   zmeta_fft_c2c.<locals>.<lambda>I  s   ø€  w¨q¡z€ r:   rÀ   )rV   rg   r^   Ú
is_complexÚcloneÚdevice_hintÚbackendsÚmklÚis_availabler˜   r£   ræ   rß   rÄ   rÆ   rÇ   ÚminÚcufft_max_ndimrÃ   )rš   r†   ÚnormalizationrÏ   rÎ   Úoutputrå   Úworking_tensorÚmax_dimsÚ	last_dimsrí   s             @r8   Úmeta_fft_c2crû   3  s[  ø€ ô 
‡L�L�—‘×&Ñ&Ô'ÙØ�z‰z‹|Ðä�4Ó˜EÒ!¬%¯.©.×*<Ñ*<×*IÑ*IÔ*KØ�~‰~˜dŸi™i›kÓ*Ð*à—	‘	“€IØ�^‰^˜IÓ&€FÜ�4Ó˜FÒ"Ü   sÓ+ˆÜ˜  y°+ÀwÔOÐOô �s“)€KØ€NØ
Ø ×'Ñ'Ó)ˆØ×ÑÓ1¸4ÐÔ@Ü”~¤s¨;Ó'7Ó8ˆØ¤ KÓ 0°8Ñ ;Ð =Ð>ˆ	ä�&˜.¨)°YÈÕPØ!Ð"?¤C¨Ó$4°xÑ$?Ð@ˆáØˆMà˜TÑ!Ø#ˆNØ—^‘^ IÓ.‰Fà%3°V�NˆFð! r:   c                 óf   — t        | «      t        kD  st        | «      dk\  r| d   dk(  r	| d   dk(  ryy)Né   r   r3   FT)rÃ   rõ   r�   s    r8   Úuse_optimized_cufft_pathrþ   ]  s3   € Ü
ˆ3ƒx”.Ò ¤S¨£X°¢]°s¸1±vÀ²{ÀsÈ1ÁvÐQRÂ{Øàr:   c                 óf  ‡— t        j                  | j                  j                  «       t	        | j                  «       «      }t	        |«      }|d   }||   dz  dz   }t	        |«      }|||<   |r|||<   t        | «      dk(  st        | «      dk(  �rz| j                  |t        j                  | j                  «      ¬«      }	| }
t        | «      dk(  rt        |«      rt        |	|
||d¬«       nàt        |«      dk(  r|n|}t        |	|
||gd¬«       t        |«      dkD  r0| j                  |t        j                  | j                  «      ¬«      }
|d d }|rx|
|	}
}	|
j                  «       Š|j                  ˆfd	„d¬
«       t        t         t        |«      «      }|t        |«      |z
  d  }t        |	|
||d¬«       |d t        |«      |z
   }|rŒx|s:|	j                  |«      ||   k7  r#|
j#                  |t         j$                  ¬«       |
}	|	S t         j&                  j(                  j+                  «       rNt-        | |d¬«      }| j                  |t        j                  | j                  «      ¬«      }	t        |	| ||d¬«      S | j                  |t        j                  | j                  «      ¬«      S )Nrœ   rý   r3   ré   Úxpur‘   Trê   c                 ó   •— ‰|    S rD   r4   rì   s    €r8   re   zmeta_fft_r2c.<locals>.<lambda>‹  s   ø€  '¨!¡*€ r:   rÀ   r«   )rà   )rV   rg   r^   Úis_floating_pointrÄ   r£   rð   r˜   rO   r   rþ   rß   rÃ   rÆ   rÇ   rô   rõ   rÊ   r°   rñ   rò   ró   ræ   )rš   r†   rö   ÚonesidedÚinput_sizesrÎ   Úlast_dimÚlast_dim_halfsizeÚonesided_sizesr÷   rø   Útarget_sizesrå   rù   rú   rí   s                  @r8   Úmeta_fft_r2cr	  d  sž  ø€ ô 
‡L�L�—‘×-Ñ-Ô.Ü�t—y‘y“{Ó#€KÜ�[Ó!€IØ�2‰w€HØ# HÑ-°Ñ2°QÑ6ÐÜ˜+Ó&€NØ0€N�8ÑáØ/ˆ	�(Ñä�4Ó˜FÒ"¤k°$Ó&7¸5Ó&@ð —‘ØœU×>Ñ>¸t¿z¹zÓJð  ó 
ˆð ˆÜ�tÓ Ò&Ô+CÀCÔ+HÜ�f˜n¨i¸ÀdÖKô ),¨C«°Aª™9¸>ˆLÜ�f˜n¨l¸X¸JÐPTÕUÜ�3‹x˜!Š|Ø!%§¡Ø¤U×%FÑ%FÀtÇzÁzÓ%Rð "0ó "�ð
 ˜c˜r˜(ˆKÙØ)7¸˜�Ø(×/Ñ/Ó1�Ø× Ñ Û,°dð !ô ô œ~¬s°;Ó/?Ó@�Ø'¬¨KÓ(8¸8Ñ(CÐ(EÐF�	ÜØ˜N¨N¸IÈtõð *Ð*G¬C°Ó,<¸xÑ,GÐH�ò ñ Ø�{‰{˜8Ó$¨	°(Ñ(;Ò;Ø×&Ñ& yÄ×@WÑ@WÐ&ÔXØ'�àˆä	�‰×	Ñ	×	(Ñ	(Ô	*ä   s¸Ô>ˆØ—‘ØœU×>Ñ>¸t¿z¹zÓJð  ó 
ˆô ˜  y°+ÀtÔLÐLð �~‰~ØœU×>Ñ>¸t¿z¹zÓJð ó 
ð 	
r:   )Ú	generatorc                óB   — t        |t        j                  | g«      «      S rD   )r!   rV   ÚSize)Únr
  rÍ   s      r8   Úmeta_randpermr  ©  s   € ä˜S¤%§*¡*¨a¨S£/Ó2Ð2r:   ©r^   r‚   rƒ   r„   c                ó6   — t        j                  | ||||¬«      S ©Nr  ©rV   rŒ   )r  r^   r‚   rƒ   r„   s        r8   Úmeta_randperm_defaultr  ®  s    € ô �;‰;Ø	�˜v¨fÀôð r:   c                óx   ‡ ‡— dŠt        j                  ‰ ‰kD  ˆ ˆfd„«       t        j                  |||||¬«      S )Nr   c                  ó   •— d‰› d‰ › �S ©Nz:random_ expects 'from' to be less than 'to', but got from=z >= to=r4   ©ÚhighÚlows   €€r8   re   zmeta_randint.<locals>.<lambda>Ê  ó   ø€ ÐLÈSÈEÐQXÐY]ÐX^Ð_€ r:   r  ©rV   rg   rŒ   )r  r£   r^   r‚   rƒ   r„   r  s   `     @r8   Úmeta_randintr  ¼  s>   ù€ ð €CÜ	‡L�LØˆs‰
Ü_ôô �;‰;Ø�E &°ÀJôð r:   c                ót   ‡ ‡— t        j                  ‰‰ kD  ˆˆ fd„«       t        j                  |||||¬«      S )Nc                  ó   •— d‰› d‰ › �S r  r4   r  s   €€r8   re   z"meta_randint_low.<locals>.<lambda>ß  r  r:   r  r  )r  r  r£   r^   r‚   rƒ   r„   s   ``     r8   Úmeta_randint_lowr  Ñ  s9   ù€ ô 
‡L�LØˆs‰
Ü_ôô �;‰;Ø�E &°ÀJôð r:   c                ó6   — t        j                  | ||||¬«      S r  r  )r£   r^   r‚   rƒ   r„   s        r8   Úmeta_rand_defaultr!  æ  s    € ô �;‰;Ø�E &°ÀJôð r:   c                 ón  ‡ ‡— t        j                  ‰ j                  «       dk\  xr ‰ j                  d   dk(  ˆ fd„«       t        j                  ‰ j                  t         j
                  k(  ˆ fd„«       t        j                  ‰dkD  ˆfd„«       ‰ j                  d d }‰ j                  ‰g|¢d‘­«      S )Nr3   rœ   rý   c                  ó"   •— d‰ j                   › �S )Nz>_philox_key_split: key must have shape (*batch, 2), got shape ©r™   ©rÁ   s   €r8   re   z'meta_philox_key_split.<locals>.<lambda>ò  s   ø€ ÐPÐQT×QZÑQZÐP[Ð\€ r:   c                  ó"   •— d‰ j                   › �S )Nz3_philox_key_split: key must have dtype uint64, got r‘   r%  s   €r8   re   z'meta_philox_key_split.<locals>.<lambda>ö  s   ø€ ÐEÀcÇiÁiÀ[ÐQ€ r:   r   c                  ó   •— d‰ › �S )Nz4_philox_key_split: num_splits must be positive, got r4   )Ú
num_splitss   €r8   re   z'meta_philox_key_split.<locals>.<lambda>ú  s   ø€ ÐFÀzÀlÐS€ r:   )rV   rg   r†   r™   r^   Úuint64r˜   )rÁ   r(  Úbatch_sizess   `` r8   Úmeta_philox_key_splitr+  î  s—   ù€ ä	‡L�LØ�‰‹	�Q‰Ò-˜3Ÿ9™9 R™=¨AÑ-Û\ôô 
‡L�LØ�	‰	”U—\‘\Ñ!ÛQôô 
‡L�LØ�Q‰ÛSôð —)‘)˜C˜R�.€KØ�=‰=˜*Ð6 {Ð6°AÑ6Ó7Ð7r:   c                 ó  ‡ — t        j                  ‰ j                  «       dk\  xr ‰ j                  d   dk(  ˆ fd„«       t        j                  ‰ j                  t         j
                  k(  ˆ fd„«       t        j                  ‰ «      S )Nr3   rœ   rý   c                  ó"   •— d‰ j                   › �S )Nz@_philox_key_fold_in: key must have shape (*batch, 2), got shape r$  r%  s   €r8   re   z)meta_philox_key_fold_in.<locals>.<lambda>  s   ø€ ÐRÐSV×S\ÑS\ÐR]Ð^€ r:   c                  ó"   •— d‰ j                   › �S )Nz5_philox_key_fold_in: key must have dtype uint64, got r‘   r%  s   €r8   re   z)meta_philox_key_fold_in.<locals>.<lambda>  s   ø€ ÐGÈÏ	É	À{ÐS€ r:   )rV   rg   r†   r™   r^   r)  r¯   )rÁ   Údatas   ` r8   Úmeta_philox_key_fold_inr0     se   ø€ ä	‡L�LØ�‰‹	�Q‰Ò-˜3Ÿ9™9 R™=¨AÑ-Û^ôô 
‡L�LØ�	‰	”U—\‘\Ñ!ÛSôô ×Ñ˜CÓ Ð r:   c           	      ó  ‡ ‡‡‡— t        j                  ‰j                  j                  ˆ ˆfd„«       t        j                  ‰j                  t         j                  k(  ˆˆ fd„«       t        j                  ‰j
                  ‰j
                  k(  ˆˆ ˆfd„«       t        j                  ‰j                  «       dk\  xr ‰j                  d   dk(  ˆˆ fd„«       ‰j                  «       dkD  ršt        j                  ‰j                  «       ‰j                  «       dz   k(  ˆˆ ˆfd„«       ‰j                  d ‰j                  «        Št        j                  t        d	„ t        ‰‰j                  «      D «       «      ˆˆ ˆfd
„«       y y )Nc                  ó&   •— ‰ › d‰j                   › �S )Nz,: self must be a floating point tensor, got r‘   )Úop_namerš   s   €€r8   re   z1_check_philox_distribution_args.<locals>.<lambda>  s   ø€ �7�)ÐGÈÏ
É
À|ÐT€ r:   c                  ó&   •— ‰› d‰ j                   › �S )Nz": key must have dtype uint64, got r‘   ©rÁ   r3  s   €€r8   re   z1_check_philox_distribution_args.<locals>.<lambda>  s   ø€ �7�)Ð=¸c¿i¹i¸[ÐI€ r:   c                  ó@   •— ‰› d‰j                   › d‰ j                   › �S )Nz/: self and key must be on the same device, got r¡   ©rƒ   ©rÁ   r3  rš   s   €€€r8   re   z1_check_philox_distribution_args.<locals>.<lambda>  s*   ø€ Øˆið Ø—;‘;�-˜u S§Z¡Z Lð2ð r:   r3   rœ   rý   c                  ó&   •— ‰› d‰ j                   › �S )Nz5: key must have shape (2,) or (*batch, 2), got shape r$  r5  s   €€r8   re   z1_check_philox_distribution_args.<locals>.<lambda>  s   ø€ ØˆiÐLÈSÏYÉYÈKÐXð r:   c                  ó@   •— ‰› d‰ j                   › d‰j                   › �S )Nz?: batched key must have ndim == output ndim + 1, got key shape z with output shape r$  r8  s   €€€r8   re   z1_check_philox_distribution_args.<locals>.<lambda>&  s,   ø€ Ø�)ð !Ø!$§¡ Ð+>¸t¿z¹z¸lðLð r:   c              3   ó:   K  — | ]  \  }}|d k(  xs ||k(  –— Œ y­w©r3   Nr4   )rr   ÚksÚsss      r8   rt   z2_check_philox_distribution_args.<locals>.<genexpr>-  s$   è ø€ ÒM©¨¨B��a‘Ò#˜2 ™8Ó#ÑMùs   ‚c                  ó>   •— ‰› dt        ‰ «      › d‰j                  › �S )Nz: key batch shape z( is not broadcastable with output shape ©rÄ   r™   )Ú	key_batchr3  rš   s   €€€r8   re   z1_check_philox_distribution_args.<locals>.<lambda>.  s-   ø€ Ø�)Ð-¬d°9«oÐ->ð ?:Ø:>¿*¹*¸ðGð r:   )
rV   rg   r^   r  r)  rƒ   r†   r™   ÚallÚzip)r3  rš   rÁ   rA  s   ```@r8   Ú_check_philox_distribution_argsrD    s  û€ Ü	‡L�LØ�
‰
×$Ñ$ÜTôô 
‡L�LØ�	‰	”U—\‘\Ñ!ÜIôô 
‡L�LØ�‰�s—z‘zÑ!õ	
ôô 
‡L�LØ�‰‹	�Q‰Ò-˜3Ÿ9™9 R™=¨AÑ-ô	
ôð ‡w�wƒy�1‚}Ü�‰Ø�G‰G‹I˜Ÿ™› a™Ñ'õô	
ð —I‘I˜l §¡£
Ð+ˆ	Ü�‰ÜÑM´#°iÀÇÁÓ2LÔMÓMõõ	
ð r:   c                 ó    — t        d| |«       | S )NÚ_philox_normal_©rD  )rš   rÁ   ÚmeanÚstds       r8   Úmeta_philox_normal_rJ  5  s   € ä#Ð$5°t¸SÔAØ€Kr:   c                 ó    — t        d| |«       | S )NÚ_philox_uniform_rG  )rš   rÁ   r  r  s       r8   Úmeta_philox_uniform_rM  ;  s   € ä#Ð$6¸¸cÔBØ€Kr:   rö   Úlastdimc                 óø  — t        j                  | j                  j                  «       t	        | «      dk(  rÕt        | j                  «       «      }|||d   <   | j                  |t        | j                  «      ¬«      }t        |«      r.t        || j                  t         j                  ¬«      ||d¬«      S t        |«      dkD  rt        | |d d dd¬«      }n | j                  t         j                  ¬«      }t        ||||d   gd¬«      S t         j                  j                   j#                  «       r€| }t        |«      dkD  r|d d }t        | ||d¬«      }|dd  }t        |j                  «       «      }|||d   <   | j                  |t        | j                  «      ¬«      }	t        |	|||d¬«      S t        | j                  «       «      }|||d   <   | j                  |t        | j                  «      ¬«      S )	Nré   rœ   r‘   r«   Frê   r3   r   )rV   rg   r^   rî   rð   rÄ   r£   r˜   r`   rþ   rß   rï   r°   rÃ   rû   rñ   rò   ró   )
rš   r†   rö   rN  rÎ   r÷   ÚtemprØ   Úc2c_dimsrÍ   s
             r8   Úmeta_fft_c2rrR  A  s»  € ô 
‡L�L�—‘×&Ñ&Ô'ä�4Ó˜FÒ"Ü˜Ÿ™›Ó%ˆ	Ø$ˆ	�#�b‘'Ñà—‘ 	´ÀÇÁÓ1L�ÓMˆä# CÔ(ÜØØ—
‘
¬×)@Ñ)@�
ÓAØØØôð ô �3‹x˜!Š|Ü# D¨#¨c¨r¨(°A¸uÔE‘à—z‘z´×0GÑ0G�zÓH�Ü˜V T¨9°s¸2±w°iÈÔOÐOä	�‰×	Ñ	×	(Ñ	(Ô	*àˆÜˆs‹8�aŠ<Ø˜3˜B�xˆHÜ   x°ÈÔNˆEØ�b�c�(ˆCä˜Ÿ™›Ó&ˆ	Ø$ˆ	�#�b‘'ÑØ�n‰n˜Y¬o¸d¿j¹jÓ.IˆnÓJˆÜ˜˜e Y°¸UÔCÐCô ˜Ÿ™›Ó%ˆ	Ø$ˆ	�#�b‘'ÑØ�~‰~˜i¬¸t¿z¹zÓ/Jˆ~ÓKÐKr:   c                 ó  — ddl m}  || «      s#t        j                  | «      dk(  rt	        d«      ‚t        |t        «      r@|j                  | |«      }t        j                  j                  || j                  «       «       | S )Nr   ©Úfree_unbacked_symbolsr3   zQmore than one element of the written-to tensor refers to a single memory location)Ú%torch.fx.experimental.symbolic_shapesrU  rV   Ú_debug_has_internal_overlapÚRuntimeErrorrp   r   ÚtoÚ_refsÚexpandr£   )rš   ÚsrcÚnon_blockingrU  Úintermediates        r8   Ú
meta_copy_r_  p  sr   € õ Lñ " $Ô'¬E×,MÑ,MÈdÓ,SÐWXÒ,XäØ_ó
ð 	
ô �#”vÔØ—v‘v˜d LÓ1ˆô 	�‰×Ñ˜<¨¯©«Ô5Ø€Kr:   c                 óö   — t        | j                  «       «      }t        | j                  «       «      }|| j                  «       k\  rdn
||   ||   z  }|j	                  |d«       |j	                  ||«       ||fS r2   )rÄ   r£   rÆ   r†   Úinsert)Útensorr†   Úresult_sizesÚresult_stridesÚ
new_strides        r8   ÚinferUnsqueezeGeometryrf  Ž  sq   € Ü˜Ÿ™›Ó&€LÜ˜&Ÿ-™-›/Ó*€Nà˜VŸZ™Z›\Ò)‘¨|¸CÑ/@À>ÐRUÑCVÑ/V€Jà×Ñ˜˜QÔà×Ñ˜#˜zÔ*Ø˜Ð'Ð'r:   c                 ó‚   — t        || j                  «       dz   «      }t        | |«      \  }}| j                  ||«       | S r2   )r¸   r†   rf  rË   )rš   r†   Úg_sizesÚ	g_stridess       r8   Úmeta_unsqueeze_rj  š  s>   € ä
˜˜dŸh™h›j¨1™nÓ
-€CÜ/°°cÓ:Ñ€GˆYØ×Ñ�W˜iÔ(Ø€Kr:   rØ   ÚweightÚ_metaÚbiasÚ_activation_optÚ	out_dtypec                 ó(  — t        | j                  «      }|�R|j                  d«      |j                  d«      k7  r/t        d|j                  d«      › d|j                  d«      › �«      ‚|j                  d«      | j                  d«      dz  k7  r2t        d|j                  d«      › d| j                  d«      dz  › �«      ‚|j                  d«      |d<   t	        | j                  «      dk7  r"t        d	t	        | j                  «      › d
�«      ‚d| j                  d«      f}|�K| j
                  t        j                  k(  r|t        j                  k(  st        d| j
                  › d|› �«      ‚| j                  |||€| j
                  n|¬«      }|S )Nr   z%output size mismatch: weight.size(0)=ú != bias.size(0)=r3   rœ   rý   zweight.size(1)=z != input.size(-1)/2=z0we can only handle the squashed input case, got úD inputzKout_dtype is only supported for i8i8->i32 linear operator, got input.dtype=ú, out_dtype=r‘   )
rÄ   r™   r£   rŠ   rÃ   r^   rV   Úint8Úint32Únew_empty_strided)	rØ   rk  rl  rm  rn  ro  Úoutput_sizesÚtransposed_stridesr÷   s	            r8   Úmeta_sparse_structured_linearry  ¢  s   € ô ˜Ÿ™Ó$€LØÐØ�;‰;�q‹>˜TŸY™Y q›\Ò)Ü Ø7¸¿¹ÀA»Ð7GÐGXÐY]×YbÑYbÐcdÓYeÐXfÐgóð ð ‡{�{�1ƒ~˜Ÿ™ B›¨!Ñ+Ò+ÜØ˜fŸk™k¨!›nÐ-Ð-BÀ5Ç:Á:ÈbÃ>ÐTUÑCUÐBVÐWó
ð 	
ð —{‘{ 1“~€L�Ñô ˆ5�;‰;Ó˜1ÒÜØ>¼sÀ5Ç;Á;Ó?OÐ>PÐPWÐXó
ð 	
ð ˜UŸZ™Z¨›]Ð+ÐàÐØ—‘œuŸz™zÒ)¨i¼5¿;¹;Ò.FÜ Ø]Ð^c×^iÑ^iÐ]jÐjvð  xAð  wBð  Cóð ð ×$Ñ$ØØØ&Ð.ˆe�kŠk°Ið %ó €Fð €Mr:   Úmat1Ú	mat1_metaÚmat2c                 ó2  — t        | j                  «      dk7  r"t        dt        | j                  «      › d�«      ‚t        |j                  «      dk7  r"t        dt        |j                  «      › d�«      ‚t        |j                  «      dk7  r"t        dt        |j                  «      › d�«      ‚| j                  d«      |j                  d«      dz  k7  r2t        d| j                  d«      › d	|j                  d«      dz  › �«      ‚| j                  d«      |j                  d«      g}|�K|j                  t
        j                  k(  r|t
        j                  k(  st        d
|j                  › d|› �«      ‚|j                  ||€|j                  n|¬«      }|S )Nrý   úmat1 must be 2D, got ÚDúmat1_meta must be 2D, got úmat2 must be 2D, got r3   r   úmat1.size(1)=ú != mat2.size(0)/2=úJout_dtype is only supported for i8i8->i32 linear operator, got mat2.dtype=rs  r‘   ©	rÃ   r™   rŠ   r£   r^   rV   rt  ru  r˜   )rz  r{  r|  ro  rw  r÷   s         r8   Úmeta_sparse_structured_mmr†  Ï  sv  € ô ˆ4�:‰:ƒ˜!ÒÜÐ4´S¸¿¹³_Ð4EÀQÐGÓHÐHÜ
ˆ9�?‰?Ó˜qÒ ÜÐ9¼#¸i¿o¹oÓ:NÐ9OÈqÐQÓRÐRÜ
ˆ4�:‰:ƒ˜!ÒÜÐ4´S¸¿¹³_Ð4EÀQÐGÓHÐHØ‡y�y�ƒ|�t—y‘y “| aÑ'Ò'ÜØ˜DŸI™I a›L˜>Ð)<¸T¿Y¹YÀq»\ÈAÑ=MÐ<NÐOó
ð 	
ð —I‘I˜a“L $§)¡)¨A£,Ð/€LàÐØ—
‘
œeŸj™jÒ(¨Y¼%¿+¹+Ò-EÜ Ø\Ð]a×]gÑ]gÐ\hÐhtÐu~Ðtð  Aóð ð �^‰^ØØ%Ð-ˆd�jŠj°9ð ó €Fð
 €Mr:   r3   )ÚalphaÚbetaro  c                óJ  — t        | j                  «      dk7  r"t        dt        | j                  «      › d�«      ‚t        |j                  «      dk7  r"t        dt        |j                  «      › d�«      ‚t        |j                  «      dk7  r"t        dt        |j                  «      › d�«      ‚t        |j                  «      dk7  r"t        dt        |j                  «      › d�«      ‚| j                  d	«      |j                  d	«      k7  r/t        d
| j                  d	«      › d|j                  d	«      › �«      ‚|j                  d«      |j                  d	«      dz  k7  r2t        d|j                  d«      › d|j                  d	«      dz  › �«      ‚|j                  d	«      |j                  d«      g}|�K|j                  t
        j                  k(  r|t
        j                  k(  st        d|j                  › d|› �«      ‚|j                  ||€|j                  n|¬«      }|S )Nr3   zKonly input broadcasted to columns of mat1 * mat2 product is supported, got rr  rý   r~  r  r€  r�  r   zUonly input broadcasted to columns of mat1 * mat2 product is supported, input.size(0)=z != mat1.size(0)=r‚  rƒ  r„  rs  r‘   r…  )	rØ   rz  r{  r|  r‡  rˆ  ro  rw  r÷   s	            r8   Úmeta_sparse_structured_addmmrŠ  ï  sÿ  € ô ˆ5�;‰;Ó˜1ÒÜØYÔZ]Ð^c×^iÑ^iÓZjÐYkÐkrÐsó
ð 	
ô ˆ4�:‰:ƒ˜!ÒÜÐ4´S¸¿¹³_Ð4EÀQÐGÓHÐHÜ
ˆ9�?‰?Ó˜qÒ ÜÐ9¼#¸i¿o¹oÓ:NÐ9OÈqÐQÓRÐRÜ
ˆ4�:‰:ƒ˜!ÒÜÐ4´S¸¿¹³_Ð4EÀQÐGÓHÐHØ‡z�z�!ƒ}˜Ÿ	™	 !›Ò$ÜðØ"ŸZ™Z¨›]˜OÐ+<¸T¿Y¹YÀq»\¸NðLó
ð 	
ð ‡y�y�ƒ|�t—y‘y “| aÑ'Ò'ÜØ˜DŸI™I a›L˜>Ð)<¸T¿Y¹YÀq»\ÈAÑ=MÐ<NÐOó
ð 	
ð —I‘I˜a“L $§)¡)¨A£,Ð/€LàÐØ—
‘
œeŸj™jÒ(¨Y¼%¿+¹+Ò-EÜ Ø\Ð]a×]gÑ]gÐ\hÐhtÐu~Ðtð  Aóð ð �^‰^ØØ%Ð-ˆd�jŠj°9ð ó €Fð
 €Mr:   Úcompressed_AÚdense_Br‡  Útranspose_resultÚalg_idÚsplit_kÚsplit_k_modec	           	      ó*  — |j                   t        j                  t        j                  t        j                  t        j
                  t        j                  hvrt        d|j                   › �«      ‚| j                   |j                   k7  r%t        d| j                   › d|j                   › �«      ‚t        |j                  «      dk7  r"t        dt        |j                  «      › d�«      ‚| j                   t        j
                  t        j                  fv }	|	r|j                  «       rt        d«      ‚|j                  d«      }
| j                  d	«      }|�4||j                  d	«      k7  r t        d
|› d|j                  d	«      › �«      ‚|�k|	r@|t        j                  t        j                  t        j                  t        j                  hv s)t        d| j                   › d|j                   › d|› d�«      ‚|r|
|fn||
f}|j                  ||¬«      S )NzA_cslt_sparse_mm only supports fp16, bf16, int8, and fp8e4m3, got z%inputs must have the same dtype, got r¡   rý   z-_cslt_sparse_mm only supports 2d inputs, got r  z.dense input must be transposed for 8bit dtypesr3   r   zbias size mismatch: m=rq  zout_dtype is not supported for z x z -> z matmul!r‘   )r^   rV   Úfloat32Úfloat16Úbfloat16rt  Úfloat8_e4m3fnrŠ   rÃ   r™   Úis_contiguousr£   ru  r˜   )r‹  rŒ  rm  r‡  ro  r�  rŽ  r�  r�  Úis_8bit_input_typer  ÚmÚoutput_shapes                r8   Úmeta__cslt_sparse_mmrš    s÷  € ð ‡}�}Ü�‰Ü�‰Ü�‰Ü�
‰
Ü×Ñðñ ô ØOÐPW×P]ÑP]ÈÐ_ó
ð 	
ð ×Ñ˜WŸ]™]Ò*ÜØ3°L×4FÑ4FÐ3GÀuÈWÏ]É]ÈOÐ\ó
ð 	
ô ˆ7�=‰=Ó˜QÒÜØ;¼CÀÇÁÓ<NÐ;OÈqÐQó
ð 	
ð &×+Ñ+´·
±
¼E×<OÑ<OÐ/PÐPÐáØ× Ñ Ô"Ü Ð!QÓRÐRà�‰�Q‹€AØ×Ñ˜!Ó€AØÐØ�—	‘	˜!“ÒÜ Ø(¨¨Ð+<¸T¿Y¹YÀq»\¸NÐKóð ð ÐáØä—‘Ü—‘Ü—‘Ü×#Ñ#ð	ñô !Ø1°,×2DÑ2DÐ1EÀSÈÏÉÈÐW[Ð\eÐ[fÐfnÐoóð ñ .�A�q‘6°A°q°6€LØ×Ñ˜\°ÐÓ;Ð;r:   T)Úinclude_selfr“   ÚsourceÚreducer›  c                óL   — t        j                  | t         j                  ¬«      S r»   ©rV   r¯   r°   ©rš   r†   r“   rœ  r�  r›  s         r8   Úmeta_index_reducer¡  [  s   € ô ×Ñ˜D´×0GÑ0GÔHÐHr:   c                ó   — | S rD   r4   r   s         r8   Úmeta_index_reduce_r£  h  s	   € ð €Kr:   c                 ó¢   — t        | j                  «       «      }| j                  «       dkD  r|j                  «       ||<   | j	                  |«      S ©Nr   )rÄ   r£   r†   r—   r˜   )rš   r†   r“   Úresult_sizes       r8   Úmeta_index_selectr§  v  s@   € ô �t—y‘y“{Ó#€KØ‡x�xƒz�A‚~Ø Ÿ;™;›=ˆ�CÑØ�>‰>˜+Ó&Ð&r:   )Úlengthsr´   ÚoffsetsÚaxisÚunsafeÚinitialr/  r¨  r©  rª  r«  c                óÄ   ‡ ‡— |�t        d«      ‚ˆˆ fd„}|� ||j                  «      S |�+|j                  d d |j                  d   dz
  fz   }	 ||	«      S t        d«      ‚)Nz?segment_reduce(): indices based reduction is not supported yet.c                 óŒ   •— t        j                  | ‰j                  ‰dz   d  z   ‰j                  dt         j                  ¬«      S )Nr3   r€   ©r^   rƒ   r¬   )rV   rŒ   r™   r^   r°   )Úlengths_shaperª  r/  s    €€r8   Úsegment_reduce_lengths_tensorz:meta_segment_reduce.<locals>.segment_reduce_lengths_tensor�  s>   ø€ Ü�{‰{Ø˜DŸJ™J t¨a¡x zÐ2Ñ2Ø—*‘*ØÜ×1Ñ1ô	
ð 	
r:   rœ   r3   z<segment_reduce(): Either lengths or offsets must be defined.)ÚNotImplementedErrorr™   rX  )
r/  r�  r¨  r´   r©  rª  r«  r¬  r±  r°  s
   `    `    r8   Úmeta_segment_reducer³    s|   ù€ ð ÐÜ!ØMó
ð 	
õ
ð ÐÙ,¨W¯]©]Ó;Ð;ð ÐàŸ™ c rÐ*¨g¯m©m¸BÑ.?À!Ñ.CÐ-EÑEˆÙ,¨]Ó;Ð;Ü
ÐUÓ
VÐVr:   c                 ó$   — | j                  d«      S ©Nr4   ©r˜   r±   s    r8   Úmeta_maxr·  £  ó   € ð �>‰>˜"ÓÐr:   c                 óÂ   — t        j                  | j                  |f«      }t        | ||«      }| j	                  |«      | j	                  |t
        j                  ¬«      fS ©Nr‘   ©rO   Úreduction_dimsr™   Ú_compute_reduction_shaper˜   rV   r•   ©rš   r†   Úkeepdimr™  s       r8   Úmeta_max_dimrÀ  ©  óR   € ä
×
Ñ
˜tŸz™z¨C¨6Ó
2€CÜ+¨D°#°wÓ?€Là�‰�|Ó$Ø�‰�|¬5¯:©:ˆÓ6ðð r:   c                 ó$   — | j                  d«      S rµ  r¶  r±   s    r8   Úmeta_minrÃ  ³  r¸  r:   c                 óÂ   — t        j                  | j                  |f«      }t        | ||«      }| j	                  |«      | j	                  |t
        j                  ¬«      fS rº  r»  r¾  s       r8   Úmeta_min_dimrÅ  ¹  rÁ  r:   c                 ó¸   — | j                  «       rt        | j                  «      }nt        | t        j
                  ¬«      \  }}t        j                  | |¬«      S ©N©rN   r‘   )rî   r   r^   r   r   ÚINT_TO_FLOATrV   r¯   )rš   rS   rR   s      r8   Ú
meta_anglerÊ  Ã  sI   € à‡�ÔÜ/°·
±
Ó;‰ä,ØÜ ?× LÑ Lô
‰ˆˆ<ô ×Ñ˜D¨Ô5Ð5r:   c                 ó¨   — t        j                  || j                  «       | j                  «       |j	                  t        j
                  | «      «      S rD   )rV   Ú_resize_output_r£   rƒ   Úcopy_Úangle)rš   rÍ   s     r8   Úmeta_angle_outrÏ  Ï  s6   € ä	×Ñ˜#˜tŸy™y›{¨D¯K©KÔ8Ø�9‰9”U—[‘[ Ó&Ó'Ð'r:   c                  ó   — y rD   r4   )Úvals    r8   Úassert_asyncrÒ  Õ  ó   € à
r:   c                  ó   — y rD   r4   )rÑ  Ú
assert_msgs     r8   Úassert_async_metarÖ  Ú  rÓ  r:   c                  ó   — y rD   r4   )Úss    r8   Ú
print_metarÙ  ß  rÓ  r:   ©r^   r‚   rƒ   r„   r¬   c                 ó0   — t        j                  dd¬«      S )Nr4   r€   r7  r  rÚ  s        r8   Úmake_dep_tokenrÜ  ä  s   € ô �;‰;�r &Ô)Ð)r:   c                 óh   — ddl m} t        | t        t        f«      rt        d«      ‚ || ||¬«       y )Nr   )Úconstrain_rangeú'Constraining SymFloat or Symbool is nyi©rô   Úmax)rV  rÞ  rp   r
   r	   Ú
ValueError)r£   rô   rá  rÞ  s       r8   Úsym_constrain_rangerã  ð  s/   € õ Fä�$œ¤7Ð+Ô,ÜÐBÓCÐCÙ�D˜c sÖ+r:   c                 ó6   — t         j                  | ||¬«       |S ©Nrà  )r-   rã  ©r£   rô   rá  Ú	dep_tokens       r8   Úfunctional_sym_constrain_rangerè  ú  s   € ä×Ñ˜T s°ÐÔ4ØÐr:   c                 ó.  — ddl m} |€|€t        j                  | dk\  «       y t	        | t
        t        f«      rt        d«      ‚t        | «      t        u r5|�t        j                  | |k\  «       |�t        j                  | |k  «       y  || ||¬«       y )Nr   )Ú_constrain_range_for_sizerß  rà  )
rV  rê  rV   rg   rp   r
   r	   râ  rz   rã   )r£   rô   rá  rê  s       r8   Úsym_constrain_range_for_sizerë     s„   € õ Pà
€{�s�{Ü�‰�T˜Q‘YÔØä�$œ¤7Ð+Ô,ÜÐBÓCÐCÜˆDƒz”SÑØˆ?Ü�L‰L˜ ™Ô%Øˆ?Ü�L‰L˜ ™Ô%ØÙ˜d¨°Ö5r:   c                 ó6   — t         j                  | ||¬«       |S rå  )r-   rë  ræ  s       r8   Ú'functional_sym_constrain_range_for_sizerí    s   € ä×%Ñ% d°¸Ð%Ô=ØÐr:   c                 ó   — |S rD   r4   )rÑ  rÕ  rç  s      r8   Úfunctional_assert_async_metarï    s   € àÐr:   Úf_namec                 óà   ‡ ‡— ‰ j                  «       dk  rt        ‰› d‰ j                  «       › �«      ‚t        j                  ‰ j	                  d«      ‰ j	                  d«      k(  ˆˆ fd„«       y )Nrý   z8: The input tensor must have at least 2 dimensions, got rœ   éþÿÿÿc                  óV   •— ‰ › d‰j                  d«      › d‰j                  d«      › d�S )Nz5: A must be batches of square matrices, but they are rò  ú by rœ   ú	 matricesr¢   )rð  rš   s   €€r8   re   z#squareCheckInputs.<locals>.<lambda>(  s1   ø€ �6�(ð ØŸ	™	 "›� d¨4¯9©9°R«=¨/¸ðD€ r:   )r†   rŠ   rV   rg   r£   )rš   rð  s   ``r8   r­   r­      s^   ù€ Ø‡x�xƒz�A‚~ÜØˆhÐNÈtÏxÉxËzÈlÐ[ó
ð 	
ô 
‡L�LØ�	‰	�"‹˜Ÿ™ 2›Ñ&ô	Dõr:   ÚAÚnamec                 óº  ‡ ‡‡— t        j                  ‰ j                  ‰j                  k(  ˆˆ fd„«       t        j                  ‰ j                  ‰j                  k(  ˆˆ fd„«       t        j                  ‰j	                  d«      ‰j	                  d«      k(  ˆfd„«       t        j                  ‰j	                  d«      ‰ j	                  d«      k(  ˆˆˆ fd„«       y )Nc                  ó>   •— d‰j                   › d‰ j                   › d�S )Nz:Expected b and A to be on the same device, but found b on z
 and A on ú	 instead.r7  ©rö  rš   s   €€r8   re   z(linearSolveCheckInputs.<locals>.<lambda>3  s%   ø€ ØHØ�{‰{ˆm˜: a§h¡h Z¨yð:ð r:   c                  ó>   •— d‰j                   › d‰ j                   › d�S )Nz=Expected b and A to have the same dtype, but found b of type z and A of type rú  r‘   rû  s   €€r8   re   z(linearSolveCheckInputs.<locals>.<lambda>;  s%   ø€ ØKØ�z‰zˆl˜/¨!¯'©'¨°)ð=ð r:   rœ   rò  c                  óR   •— d‰ j                  d«      › d‰ j                  d«      › d�S )Nz3A must be batches of square matrices, but they are rò  rô  rœ   rõ  r¢   ©rö  s   €r8   re   z(linearSolveCheckInputs.<locals>.<lambda>C  s0   ø€ ðØŸF™F 2›J˜< t¨A¯F©F°2«J¨<°yðBð r:   c                  óž   •— d‰› d‰ j                  d«      › d‰ j                  d«      › d‰j                  d«      › d‰j                  d«      › �
S )NzIncompatible matrix sizes for z: each A matrix is rœ   rô  z but each b matrix is rò  r¢   )rö  r÷  rš   s   €€€r8   re   z(linearSolveCheckInputs.<locals>.<lambda>K  sR   ø€ Ø,¨T¨Fð 3ØŸ™ ›˜ D¨¯©°«¨Ø$ T§Y¡Y¨r£] O°4¸¿	¹	À"»°ðHð r:   )rV   rg   rƒ   r^   r£   )rš   rö  r÷  s   ```r8   ÚlinearSolveCheckInputsr   0  s�   ú€ Ü	‡L�LØ�‰�q—x‘xÑô	
ôô 
‡L�LØ�
‰
�a—g‘gÑô	
ôô 
‡L�LØ	�‰ˆr‹
�a—f‘f˜R“jÑ ó	
ôô 
‡L�LØ	�‰ˆr‹
�d—i‘i “mÑ#õ	
õr:   ÚtÚallow_low_precision_dtypesc                 óJ  ‡‡— | j                   Št        j                  | j                  «       xs | j	                  «       ˆˆfd„«       |sYt        j                  ‰t        j
                  t        j                  t        j                  t        j                  fv ˆˆfd„«       y y )Nc                  ó   •— ‰› d‰ › �S )Nz<: Expected a floating point or complex tensor as input. Got r4   ©r^   rð  s   €€r8   re   z(checkFloatingOrComplex.<locals>.<lambda>\  s   ø€ �6�(ÐVÐW\ÐV]Ð^€ r:   c                  ó   •— ‰› d‰ › �S )Nz*: Low precision dtypes not supported. Got r4   r  s   €€r8   re   z(checkFloatingOrComplex.<locals>.<lambda>a  s   ø€ �v�hÐHÈÈÐP€ r:   )	r^   rV   rg   r  rî   rZ   r\   rY   r[   )r  rð  r  r^   s    ` @r8   r®   r®   T  sn   ù€ ð
 �G‰G€EÜ	‡L�LØ	×ÑÓÒ/ §¡£Ü^ôñ &Ü�‰Ø”e—k‘k¤5§<¡<´·±¼u¿}¹}ÐMÐMÜPõ	
ð &r:   Úarg_namec                 ó^   ‡‡— t        j                  | j                  «       dk\  ˆˆfd„«       y )Nrý   c                  ó   •— ‰› d‰ › d�S )Nz: The input tensor z! must have at least 2 dimensions.r4   )r  rð  s   €€r8   re   zcheckIsMatrix.<locals>.<lambda>i  s   ø€ �6�(Ð-¨h¨ZÐ7XÐY€ r:   )rV   rg   r†   )rö  rð  r  s    ``r8   ÚcheckIsMatrixr
  f  s   ù€ Ü	‡L�LØ	�‰‹�1‰ÜYõr:   ÚBrÕ   c                 ó   ‡ ‡‡‡— t        ‰ ‰«       t        ‰‰«       t        j                  ‰r#‰ j	                  d«      ‰j	                  d«      k(  n"‰ j	                  d«      ‰j	                  d«      k(  ˆ ˆˆˆfd„«       y )Nrò  rœ   c                  ó¬   •— ‰› d‰rdnd› d‰ j                  d«      › d‰ j                  d«      › d‰j                  d«      › d‰j                  d«      › d	�S )
Nz2: Incompatible shapes of A and B for the equation zAX = BzXA = Bz (rò  r=   rœ   r¡   ry   r¢   )rö  r  rð  rÕ   s   €€€€r8   re   z#checkInputsSolver.<locals>.<lambda>r  s[   ø€ ØˆhÐHÙ‰x XÐ.Ø�—‘˜“�˜A˜aŸf™f R›j˜\¨¨q¯v©v°b«z¨l¸!¸A¿F¹FÀ2»J¸<ÀqðJð r:   )r­   r
  rV   rg   r£   )rö  r  rÕ   rð  s   ````r8   ÚcheckInputsSolverr  m  sY   û€ Ü�a˜Ô Ü�!�VÔÜ	‡L�LÙ$(ˆ�‰ˆr‹
�a—f‘f˜R“jÒ ¨a¯f©f°R«j¸A¿F¹FÀ2»JÑ.Fö	
õr:   ÚresultÚfn_nameÚresult_namec                 ór   ‡ ‡‡‡— t        j                  ‰j                  ‰j                  k(  ˆ ˆˆˆfd„«       y )Nc            	      óL   •— ‰ › d‰› d‰› d‰j                   › d‰j                   › �	S )Nz: Expected z5 and input tensors to be on the same device, but got z on z and input on r7  )r  rØ   r  r  s   €€€€r8   re   z!checkSameDevice.<locals>.<lambda>‚  s5   ø€ Øˆi�{ ; -Ð/dØˆm˜4 §¡˜¨n¸U¿\¹\¸NðLð r:   )rV   rg   rƒ   )r  r  rØ   r  s   ````r8   ÚcheckSameDevicer  z  s&   û€ ô 
‡L�LØ�‰˜Ÿ™Ñ%ö	
õr:   ÚUPLOc                 óŒ   ‡ — ‰ j                  «       }t        j                  t        ‰ «      dk(  xr |dk(  xs |dk(  ˆ fd„«       y )Nr3   ÚUÚLc                  ó   •— d‰ › �S )Nz1Expected UPLO argument to be 'L' or 'U', but got r4   )r  s   €r8   re   zcheckUplo.<locals>.<lambda>�  s   ø€ ÐCÀDÀ6ÐJ€ r:   )ÚupperrV   rg   rÃ   )r  ÚUPLO_uppercases   ` r8   Ú	checkUplor  ‰  s<   ø€ Ø—Z‘Z“\€NÜ	‡L�LÜˆD‹	�Q‰ÒK˜N¨cÑ1ÒJ°^ÀsÑ5JÛJõr:   ÚeigenvaluesÚeigenvectorsr  Ú	compute_vc                 óT  — t        | d«       t        |«       t        | j                  «      }|r/| j	                  |«      }|j                  |t        |d¬«      «       n| j	                  dg«      }|j                  «        | j	                  |t        | j                  «      ¬«      }||fS )Nzlinalg.eighF©Ú	row_majorr   r‘   )
r­   r  rÄ   r™   r˜   rË   r   Úpopr`   r^   )rö  r  r  r™   ÚvecsÚvalss         r8   Úmeta__linalg_eighr&  ‘  s‰   € ô �a˜Ô'Üˆd„Oä�—‘‹M€EÙØ�{‰{˜5Ó!ˆØ×Ñ˜Ô ;¸EÈUÔ SÕTà�{‰{˜A˜3Óˆà	‡I�I„KØ�;‰;�u¤O°A·G±GÓ$<ˆ;Ó=€Dà�ˆ:Ðr:   c                 óî   — t        | d«       t        j                  | j                  «      r| j                  nt        j                  | j                  «      }| j                  | j                  d d |¬«      S )Nzlinalg.eigvalsrœ   r‘   )r­   rO   r‰   r^   r   r˜   r™   )rØ   Úcomplex_dtypes     r8   Úmeta__linalg_eigvalsr)  ¤  sc   € ô �eÐ-Ô.ô ×!Ñ! %§+¡+Ô.ð 	�Šä×.Ñ.¨u¯{©{Ó;ð ð
 �?‰?˜5Ÿ;™; s¨Ð+°=ˆ?ÓAÐAr:   c                 ó®  — t        | d«       t        j                  | j                  «      r| j                  nt        j                  | j                  «      }| j                  | j                  d d |¬«      }| j                  | j                  |¬«      }t        | «      dk(  }|j                  | j                  t        | j                  |¬«      «       ||fS )Nz
linalg.eigrœ   r‘   ré   r!  )
r­   rO   r‰   r^   r   r˜   r™   rð   rË   r   )rØ   r(  r³   ÚvectorsÚis_cudas        r8   Úmeta_linalg_eigr-  °  sµ   € ô �e˜\Ô*ô ×!Ñ! %§+¡+Ô.ð 	�Šä×.Ñ.¨u¯{©{Ó;ð ð
 �_‰_˜UŸ[™[¨¨"Ð-°]ˆ_ÓC€FØ�o‰o˜eŸk™k°ˆoÓ?€GÜ˜%Ó  FÑ*€GØ×ÑØ�‰Ô0°·±ÈÔPôð �7ˆ?Ðr:   r\  c                 óv   — | j                   j                  t        j                  ¬«      j	                  dd«      S )Nr«   rò  rœ   )ÚmTrï   rV   r°   Ú	transpose)r\  s    r8   ÚcloneBatchedColumnMajorr1  Â  s*   € Ø�6‰6�<‰<¤e×&=Ñ&=ˆ<Ó>×HÑHÈÈRÓPÐPr:   r  c                 ó   — t        | «      S rD   )r1  )rš   rö  r  s      r8   Ú_cholesky_solve_helperr3  Æ  s   € ô # 4Ó(Ð(r:   c                 óØ   ‡ ‡— t        j                  ‰ j                  dk\  ˆ fd„«       t        j                  ‰j                  dk\  ˆfd„«       t        ‰ ‰d«      \  }}t	        |||«      S )Nrý   c                  ó$   •— d‰ j                   › d�S )Nz-b should have at least 2 dimensions, but has ú dimensions instead©r¥   r±   s   €r8   re   z cholesky_solve.<locals>.<lambda>Ñ  s   ø€ Ð?ÀÇ	Á	¸{ÐJ]Ð^€ r:   c                  ó$   •— d‰ j                   › d�S )Nz-u should have at least 2 dimensions, but has r6  r7  rþ  s   €r8   re   z cholesky_solve.<locals>.<lambda>Õ  s   ø€ Ð?ÀÇÁ¸xÐGZÐ[€ r:   Úcholesky_solve)rV   rg   r¥   Ú!_linalg_broadcast_batch_dims_namer3  )rš   rö  r  Úself_broadcastedÚA_broadcasteds   ``   r8   r9  r9  Ì  sh   ù€ ô 
‡L�LØ�	‰	�Q‰Û^ôô 
‡L�LØ	�‰�!‰Û[ôô 'HØˆaÐ!ó'Ñ#Ð�mô "Ð"2°MÀ5ÓIÐIr:   c                 ó    — | j                  «       dk(  r%t        j                  | t        j                  ¬«      S t	        | d«       t        | «      S )Nr   r«   Úcholesky)r—   rV   r¯   Úlegacy_contiguous_formatr­   r1  ©rš   r  s     r8   r>  r>  Ý  s@   € ð ‡z�zƒ|�qÒÜ×Ñ ´E×4RÑ4RÔSÐSÜ�d˜JÔ'Ü" 4Ó(Ð(r:   c                 ó0   — t        | d«       t        | «      S )NÚcholesky_inverse)r­   r1  r@  s     r8   rB  rB  æ  s   € ô �dÐ.Ô/Ü" 4Ó(Ð(r:   Úcheck_errorsc                 ó  — t        | d«       t        | d«       | j                  }t        |«      }t	        |d«      }| j                  |«      }|j                  ||«       | j                  |d|dz
   t        j                  ¬«      }||fS )Nzlinalg.choleskyFr   rý   r‘   )	r­   r®   r™   rÃ   r   r˜   rË   rV   ru  )rö  r  rC  ÚA_shaper¥   Ú	L_stridesr  Úinfoss           r8   Úlinalg_cholesky_exrH  î  s€   € ä�aÐ*Ô+Ü˜1Ð/Ô0à�g‰g€GÜˆw‹<€Dô ,¨G°UÓ;€IØ	�‰�GÓ€AØ‡M�M�'˜9Ô%ð �K‰K˜  D¨1¡HÐ-´U·[±[ˆKÓA€EØˆeˆ8€Or:   Útauc                 ó@  ‡ ‡‡— t        j                  ‰ j                  dk\  d„ «       t        j                  ‰ j                  d«      ‰ j                  d«      k\  d„ «       t        j                  ‰ j                  d«      ‰j                  d«      k\  d„ «       t        j                  ‰ j                  ‰j                  z
  dk(  ˆ ˆfd„«       ‰ j                  dkD  r:‰ j                  d d }‰j                  d d Št        j                  ‰|k(  ˆfd	„«       t        j                  ‰j
                  ‰ j
                  k(  ˆ ˆfd
„«       t        d‰‰ d«       t        j                  ‰ j                  t        ‰ j                  d¬«      ‰ j
                  ‰ j                  ¬«      S )Nrý   c                   ó   — y)NzHtorch.linalg.householder_product: input must have at least 2 dimensions.r4   r4   r:   r8   re   z,linalg_householder_product.<locals>.<lambda>  rm   r:   rò  rœ   c                   ó   — y)Nzbtorch.linalg.householder_product: input.shape[-2] must be greater than or equal to input.shape[-1]r4   r4   r:   r8   re   z,linalg_householder_product.<locals>.<lambda>  rm   r:   c                   ó   — y)Nz`torch.linalg.householder_product: input.shape[-1] must be greater than or equal to tau.shape[-1]r4   r4   r:   r8   re   z,linalg_householder_product.<locals>.<lambda>  rm   r:   r3   c                  ó<   •— d‰j                   › d‰ j                   › �S )Nzptorch.linalg.householder_product: Expected tau to have one dimension less than input, but got tau.ndim equal to ú and input.ndim is equal to r7  ©rØ   rI  s   €€r8   re   z,linalg_householder_product.<locals>.<lambda>  ó'   ø€ ð)Ø),¯©¨
Ð2NÈuÏzÉzÈlð\ð r:   c                  ó   •— d‰ › �S )Nzltorch.linalg.householder_product: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r4   ©Úactual_batch_tau_shapes   €r8   re   z,linalg_householder_product.<locals>.<lambda>  ó   ø€ ð6Ø6LÐ5MðOð r:   c                  ó<   •— d‰j                   › d‰ j                   › �S )Nz,torch.linalg.householder_product: tau dtype z does not match input dtype r‘   rP  s   €€r8   re   z,linalg_householder_product.<locals>.<lambda>&  s#   ø€ Ø:¸3¿9¹9¸+Ø*¨5¯;©;¨-ð9ð r:   z torch.linalg.householder_productrI  Fr!  ©r£   rÆ   r^   rƒ   )
rV   rg   r¥   r£   r™   r^   r  Úempty_stridedr   rƒ   )rØ   rI  Úexpected_batch_tau_shaperT  s   `` @r8   Úlinalg_householder_productrZ     sK  ú€ ô
 
‡L�LØ�
‰
�a‰ÙZôô 
‡L�LØ�
‰
�2‹˜%Ÿ*™* R›.Ñ(Ùtôô 
‡L�LØ�
‰
�2‹˜#Ÿ(™( 2›,Ñ&Ùrôô
 
‡L�LØ�
‰
�S—X‘XÑ Ñ"ô	
ôð ‡z�z�A‚~Ø#(§;¡;¨s°Ð#3Ð Ø!$§¡¨3¨B ÐÜ�‰Ø"Ð&>Ñ>óô	
ô 
‡L�LØ�	‰	�U—[‘[Ñ ô	
ôô Ð6¸¸UÀEÔJä×ÑØ�[‰[Ü*¨5¯;©;À%ÔHØ�k‰kØ�|‰|ô	ð r:   c                 ó2  — t        | d«       t        | dd¬«       | j                  | j                  «      }|j	                  | j                  t        | j                  d¬«      «       | j                  | j                  d d t        j                  ¬«      }||fS )Nzlinalg.inv_exF)r  r!  rò  r‘   ©r­   r®   r˜   r™   rË   r   rV   ru  )rö  rC  r  rG  s       r8   Úlinalg_inv_ex_metar]  6  sq   € ä�a˜Ô)Ü˜1˜oÈ%ÕPà	�‰�A—G‘GÓ€AØ‡M�M�!—'‘'Ô6°q·w±wÈ%ÔPÔQà�K‰K˜Ÿ™  ˜¬E¯K©KˆKÓ8€EØˆeˆ8€Or:   ÚLDÚpivotsÚinfo)Ú	hermitianrC  ra  c                óŒ  — t        | d«       t        | d«       t        j                  | j                  t        | j                  d¬«      | j                  | j                  ¬«      }| j                  | j                  d d t        j                  ¬«      }| j                  | j                  d d t        j                  ¬«      }|||fS )Nztorch.linalg.ldl_factor_exFr!  rW  rœ   r‘   rò  )
r­   r®   rV   rX  r™   r   r^   rƒ   r˜   rã   )rš   ra  rC  r^  r_  r`  s         r8   Úlinalg_ldl_factor_ex_metarc  B  sœ   € ô �dÐ8Ô9Ü˜4Ð!=Ô>Ü	×	Ñ	Ø�Z‰ZÜ*¨4¯:©:ÀÔGØ�j‰jØ�{‰{ô	
€Bð �^‰^˜DŸJ™J s¨˜O´5·9±9ˆ^Ó=€FØ�>‰>˜$Ÿ*™* S b˜/´·±ˆ>Ó;€DØˆv�tÐÐr:   )ra  c                ód  ‡ ‡‡— t        ‰ d«       t        ‰ d«       t        ‰‰ d«       t        j                  ‰j
                  dk\  ˆfd„«       ‰ j                  d d }t        j                  |‰j                  k(  ˆfd„«       t        j                  t        j                  ‰j                  «      ˆfd„«       t        j                  ‰ j                  ‰j                  k(  ˆˆ fd„«       t        ‰‰ «      \  }}t        j                  |t        |d¬	«      ‰j                  ‰j                  ¬
«      S )Nztorch.linalg.ldl_solverý   c                  ó$   •— d‰ j                   › d�S )NzMtorch.linalg.ldl_solve: Expected B to have at least 2 dimensions, but it has r6  r7  )r  s   €r8   re   z'linalg_ldl_solve_meta.<locals>.<lambda>e  ó   ø€ ðØŸ&™&˜Ð!4ð6ð r:   rœ   c                  ó$   •— d‰ j                   › d�S )Nzjtorch.linalg.ldl_solve: Expected LD.shape[:-1] and pivots.shape to be the same, but got pivots with shape ú insteadr$  ©r_  s   €r8   re   z'linalg_ldl_solve_meta.<locals>.<lambda>m  ó   ø€ ð)Ø)/¯©¨°hð@ð r:   c                  ó"   •— d‰ j                   › �S )Nz<torch.linalg.ldl_solve: Expected pivots to be integers. Got r‘   ri  s   €r8   re   z'linalg_ldl_solve_meta.<locals>.<lambda>t  s   ø€ ÐNÈvÏ|É|ÈnÐ]€ r:   c                  ó<   •— d‰j                   › d‰ j                   › �S )Nz!torch.linalg.ldl_solve: LD dtype z does not match b dtype r‘   )r  r^  s   €€r8   re   z'linalg_ldl_solve_meta.<locals>.<lambda>x  s"   ø€ Ð3°B·H±H°:Ð=UÐVW×V]ÑV]ÐU^Ð_€ r:   Fr!  rW  )r­   r®   r   rV   rg   r¥   r™   rO   Úis_integer_dtyper^   Ú_linalg_broadcast_batch_dimsrX  r   rƒ   )r^  r_  r  ra  Úexpected_pivots_shapeÚB_broadcast_sizerR   s   ```    r8   Úlinalg_ldl_solve_metarq  W  sú   ú€ ô �bÐ2Ô3Ü˜2Ð7Ô8Ü˜1˜bÐ":Ô;Ü	‡L�LØ	�‰�!‰ó	
ôð ŸH™H S b˜MÐÜ	‡L�LØ §¡Ñ-ó	
ôô 
‡L�LÜ×Ñ˜vŸ|™|Ó,Û]ôô 
‡L�LØ
�‰�A—G‘GÑÜ_ôô 7°q¸"Ó=ÑÐ�aÜ×ÑØÜ*Ð+;ÀuÔMØ�g‰gØ�x‰xô	ð r:   ÚPr  )Úpivotrs  c                óh  ‡ — t        j                  ‰ j                  dk\  ˆ fd„«       t        ‰ j                  «      }|d   }|d   }t        ||«      }||d<   |r‰ j                  |«      }n‰ j                  dg«      }||d<   ‰ j                  |«      }||d<   ||d<   ‰ j                  |«      }|||fS )Nrý   c                  ó$   •— d‰ j                   › d�S )Nz@linalg.lu: Expected tensor with 2 or more dimensions. Got size: rh  r$  rþ  s   €r8   re   z linalg_lu_meta.<locals>.<lambda>ˆ  s   ø€ ÐRÐST×SZÑSZÐR[Ð[cÐd€ r:   rò  rœ   r   )rV   rg   r¥   rÄ   r™   r   r˜   )	rö  rs  Úsizesr˜  r  Úkrr  r  r  s	   `        r8   Úlinalg_lu_metarx  ƒ  s·   ø€ ô 
‡L�LØ	�‰�!‰Ûdôô
 �—‘‹M€EØˆb‰	€AØˆb‰	€Aä��1‹€Aà€Eˆ"�IÙØ�K‰K˜Ó‰à�K‰K˜˜Óˆà€Eˆ"�IØ	�‰�EÓ€Aà€Eˆ"�IØ€Eˆ"�IØ	�‰�EÓ€AØˆa�ˆ7€Nr:   ÚLU)rs  rC  c                óê  ‡ — t        j                  ‰ j                  dk\  ˆ fd„«       t        ‰ j                  «      }|d   }|d   }t        j
                  |t        |d¬«      ‰ j                  ‰ j                  ¬«      }|j                  «        t        ||«      |d<   ‰ j                  |t         j                  ¬«      }|j                  «        ‰ j                  |t         j                  ¬«      }|||fS )	Nrý   c                  ó$   •— d‰ j                   › d�S )NzFtorch.lu_factor: Expected tensor with 2 or more dimensions. Got size: rh  r$  rþ  s   €r8   re   z*linalg_lu_factor_ex_meta.<locals>.<lambda>ª  s   ø€ ÐXÐYZ×Y`ÑY`ÐXaÐaiÐj€ r:   rò  rœ   Fr!  rW  r‘   )rV   rg   r¥   rÄ   r™   rX  r   r^   rƒ   r#  r   r˜   rã   )	rö  rs  rC  rv  r˜  r  ry  r_  r`  s	   `        r8   Úlinalg_lu_factor_ex_metar|     sÉ   ø€ ô 
‡L�LØ	�‰�!‰Ûjôô
 �—‘‹M€EØˆb‰	€AØˆb‰	€Aä	×	Ñ	ØÜ*¨5¸EÔBØ�g‰gØ�x‰xô	
€Bð 
‡I�I„Kä˜˜1“€Eˆ"�IØ�[‰[˜¤e§i¡iˆ[Ó0€Fð 
‡I�I„KØ�;‰;�u¤E§I¡Iˆ;Ó.€Dàˆv�tÐÐr:   )rÕ   Úadjointr}  c                óî  ‡ ‡‡— t        ‰ d«       t        j                  ‰ j                  ‰j                  k(  ˆˆ fd„«       t        j                  ‰j                  t        j                  k(  d„ «       t        ‰ d«       t        ‰ ‰|d«       t        j                  ‰ j                  d«      ‰j                  d«      k(  d„ «       t        j                  ‰ j                  d d ‰j                  k(  ˆfd„«       t        ‰‰ «      \  }}t        j                  |t        || ¬«      ‰j                  ‰j                  ¬	«      }|j                  «       d
k7  r"|s |j                  «       r|j                  «       }|S )Nztorch.linalg.lu_solvec                  ó>   •— d‰j                   › d‰ j                   › d�S )NzPlinalg.lu_solve: Expected LU and B to have the same dtype, but found LU of type ú and B of type rh  r‘   )r  ry  s   €€r8   re   z&linalg_lu_solve_meta.<locals>.<lambda>Ó  s(   ø€ ð$Ø$&§H¡H :¨_¸Q¿W¹W¸IÀXðOð r:   c                   ó   — y)NzElinalg.lu_solve: pivots should be a Tensor of scalar type torch.int32r4   r4   r:   r8   re   z&linalg_lu_solve_meta.<locals>.<lambda>Ú  rm   r:   zlinalg.lu_solverœ   c                   ó   — y)NzYlinalg.lu_solve: Number of pivots per batch should be same as the dimension of the matrixr4   r4   r:   r8   re   z&linalg_lu_solve_meta.<locals>.<lambda>â  rm   r:   c                  ó$   •— d‰ j                   › d�S )Nzclinalg.lu_solve: Expected LU.shape[:-1] and pivots.shape to be the same, but got pivots with shape rh  r$  ri  s   €r8   re   z&linalg_lu_solve_meta.<locals>.<lambda>è  rj  r:   r!  rW  r   )r®   rV   rg   r^   rã   r­   r  r£   r™   rn  rX  r   rƒ   r—   rî   Úconj)ry  r_  r  rÕ   r}  rp  rR   r  s   ```     r8   Úlinalg_lu_solve_metar…  Å  s.  ú€ ô ˜2Ð6Ô7Ü	‡L�LØ
�‰�A—G‘GÑô	
ôô 
‡L�LØ�‰œŸ	™	Ñ!ÙWôô �bÐ1Ô2Ü�b˜!˜TÐ#4Ô5Ü	‡L�LØ
�‰�‹�v—{‘{ 2“Ñ&Ùkôô 
‡L�LØ
�‰��"ˆ˜Ÿ™Ñ%ó	
ôô 7°q¸"Ó=ÑÐ�aä× Ñ ØÜ*Ð+;È4ÀxÔPØ�g‰gØ�x‰xô	€Fð ‡|�|ƒ~˜Ò¡4Ø×ÑÔØ—[‘[“]ˆFà€Mr:   Úunpack_dataÚunpack_pivotsc                 ó  ‡ — t        j                  ‰ j                  dk\  ˆ fd„«       |r2t        j                  |j                  t         j                  k(  d„ «       t        ‰ j                  «      }|d   }|d   }t        ||«      }||d<   |r‰ j                  |«      }n‰ j                  dg«      }|r2||d<   ‰ j                  |«      }	||d<   ||d<   ‰ j                  |«      }
n$‰ j                  dg«      }	‰ j                  dg«      }
||	|
fS )Nrý   c                  ó$   •— d‰ j                   › d�S )NzFtorch.lu_unpack: Expected tensor with 2 or more dimensions. Got size: rh  r$  )ry  s   €r8   re   z lu_unpack_meta.<locals>.<lambda>  s   ø€ ÐXÐY[×YaÑYaÐXbÐbjÐk€ r:   c                   ó   — 	 y)Nz´torch.lu_unpack: LU_pivots is expected to be a contiguous tensor of torch.int32 dtype.
Note: this function is intended to be used with the output produced by torch.linalg.lu_factorr4   r4   r:   r8   re   z lu_unpack_meta.<locals>.<lambda>  s   € ðpð r:   rò  rœ   r   )	rV   rg   r¥   r^   ru  rÄ   r™   rô   r˜   )ry  r_  r†  r‡  rv  r˜  r  rw  rr  r  r  s   `          r8   Úlu_unpack_metar‹  þ  s   ø€ ô 
‡L�LØ
�‰�1‰Ûkôñ Ü�‰Ø�L‰LœEŸK™KÑ'ñô	
ô �—‘‹N€EØˆb‰	€AØˆb‰	€AÜˆAˆq‹	€AØ€Eˆ"�IÙØ�L‰L˜Ó‰à�L‰L˜!˜ÓˆÙØˆˆb‰	Ø�L‰L˜ÓˆØˆˆb‰	Øˆˆb‰	Ø�L‰L˜Ó‰à�L‰L˜!˜ÓˆØ�L‰L˜!˜ÓˆØˆa�ˆ7€Nr:   Úmodec                 óŒ   ‡ — ‰ dk(  rd}d}||fS ‰ dk(  rd}d}||fS ‰ dk(  rd}d}||fS t        j                  dˆ fd„«       fS )NÚreducedTÚcompleteFÚrc                  ó   •— d‰ › d�S )Nzqr received unrecognized mode 'z=' but expected one of 'reduced' (default), 'r', or 'complete'r4   )rŒ  s   €r8   re   z _parse_qr_mode.<locals>.<lambda>5  s   ø€ Ø1°$°ð 8Nð Oð r:   ©rV   rg   )rŒ  Ú	compute_qrŽ  s   `  r8   Ú_parse_qr_moder”  (  s†   ø€ ØˆyÒØˆ	Øˆð �gÐÐð 
�Ò	Øˆ	Øˆð �gÐÐð 
�ŠØˆ	Øˆð �gÐÐô 	�‰Øóô	
ð �gÐÐr:   ÚQÚRc                 ó  — t        | d«       t        | d«       t        |«      \  }}| j                  d   }| j                  d   }t	        ||«      }|rMt        | j                  «      }|r|n||d<   | j                  |«      }|j                  |t        |d¬«      «       n| j                  dg«      }t        | j                  «      }	|s|s|n||	d<   | j                  |	«      }
|
j                  |	t        |	d¬«      «       ||
fS )Nz	linalg.qrrò  rœ   Fr!  r   )	r
  r®   r”  r™   rô   rÄ   r˜   rË   r   )rö  rŒ  r“  Úreduced_moder˜  r  rw  ÚQ_shaper•  ÚR_shaper–  s              r8   Úlinalg_qr_metar›  =  sè   € ô �!�[Ô!Ü˜1˜kÔ*ä,¨TÓ2Ñ€Iˆ|à	�‰�‰€AØ	�‰�‰€AÜˆAˆq‹	€AáÜ�q—w‘w“-ˆÙ'‘a¨Qˆ�‰Ø�K‰K˜Ó ˆØ	�‰�gÔ:¸7ÈeÔTÕUà�K‰K˜˜Óˆô �1—7‘7‹m€GÙ#©9‘!¸!€GˆB�KØ	�‰�GÓ€AØ‡M�M�'Ô6°wÈ%ÔPÔQØˆaˆ4€Kr:   ÚsignÚ	logabsdetc                 óˆ  — t        | d«       t        | dd«       | j                  }| j                  |d d «      }| j                  |d d t	        | j
                  «      ¬«      }t        j                  |t        |d«      | j
                  | j                  ¬«      }| j                  |d d t        j                  ¬«      }||||fS )Nzlinalg.slogdetFrò  r‘   rW  rœ   )r­   r®   r™   r˜   r`   r^   rV   rX  r   rƒ   ru  )rö  r™   rœ  r�  ry  r_  s         r8   Ú_linalg_slogdetrŸ  Y  s²   € ô �aÐ)Ô*Ü˜1Ð.°Ô6Ø�G‰G€EØ�;‰;�u˜S˜b�zÓ"€DØ—‘˜E # 2˜J¬o¸a¿g¹gÓ.F�ÓG€IÜ	×	Ñ	ØÜ*¨5°%Ó8Ø�g‰gØ�x‰xô	
€Bð �[‰[˜˜s ˜¬5¯;©;ˆ[Ó7€FØ�˜B Ð&Ð&r:   Úfull_matricesÚ
compute_uvÚdriverc                 óv  — t        | d«       t        | d«       t        | j                  d d «      }| j                  d   }| j                  d   }t	        j
                  ||«      }|r�|||r|n|gz   }| j                  |«      }	|	j                  |t        |d¬«      «       ||r|n||gz   }
| j                  |
«      }t        | «      dk(  }|j                  |
t        |
|¬«      «       n$| j                  dg«      }	| j                  dg«      }| j                  ||gz   t        | j                  «      ¬«      }|	||fS )	Nz
linalg.svdrò  rœ   Fr!  ré   r   r‘   )r
  r®   rÄ   r™   rV   r   r˜   rË   r   rð   r`   r^   )rö  r   r¡  r¢  rÑ   r˜  r  rw  ÚU_shaper  ÚV_shapeÚVr,  ÚSs                 r8   Ú_linalg_svd_metar¨  m  s(  € ô �!�\Ô"Ü˜1˜lÔ+ä�a—g‘g˜c˜r�lÓ#€JØ	�‰�‰€AØ	�‰�‰€AÜ�‰�a˜Ó€AáØ ©¡1¸1Ð=Ñ=ˆØ�K‰K˜Ó ˆØ	�‰�gÔ:¸7ÈeÔTÔUà¡]¡¸¸1Ð=Ñ=ˆØ�K‰K˜Ó ˆô
 ˜a“. FÑ*ˆØ	�‰�gÔ:¸7ÈgÔVÕWð �K‰K˜˜ÓˆØ�K‰K˜˜Óˆð 	
�‰�J ! Ñ$¬O¸A¿G¹GÓ,DˆÓE€AØˆa�ˆ7€Nr:   Úarg1Úarg2c                 ó  — | j                   d d }|j                   d d }t        ||«      }t        |«      }|| j                  d«      | j                  d«      gz  }t        |«      }||j                  d«      |j                  d«      gz  }||fS )Nrò  rœ   )r™   r%   rÄ   r£   )r©  rª  Úarg1_batch_sizesÚarg2_batch_sizesÚexpand_batch_portionÚarg1_expand_sizeÚarg2_expand_sizes          r8   rn  rn  “  s—   € ð
 —z‘z # 2�ÐØ—z‘z # 2�ÐÜ,Ð-=Ð?OÓPÐäÐ0Ó1ÐØ˜Ÿ™ 2›¨¯	©	°"«Ð6Ñ6ÐäÐ0Ó1ÐØ˜Ÿ™ 2›¨¯	©	°"«Ð6Ñ6ÐØÐ-Ð-Ð-r:   c                 óÎ   — |rt        | ||«       t        | |«      \  }}|| j                  k(  r| n| j                  |«      }||j                  k(  r|n|j                  |«      }||fS rD   )r   rn  r™   r[  )r©  rª  r÷  r¯  r°  Úarg1_broadcastedÚarg2_broadcasteds          r8   r:  r:  ¤  sv   € ñ Ü˜t T¨4Ô0ä)EÀdÈDÓ)QÑ&ÐÐ&ð ! D§J¡JÒ.‰°D·K±KÐ@PÓ4Qð ð ! D§J¡JÒ.‰°D·K±KÐ@PÓ4Qð ð Ð-Ð-Ð-r:   r¤   c                 ó    — | j                   d d }|j                  dk(  xs- | j                  dz
  |j                  k(  xr |j                   |k(  }|S )Nrœ   r3   )r™   r¥   )rØ   r¤   Úexpected_batched_rhs_shapeÚvector_cases       r8   Úlinalg_solve_is_vector_rhsr·  ¸  sS   € Ø!&§¡¨S¨bÐ!1ÐØ—*‘* ‘/ò Ø�
‰
�Q‰˜%Ÿ*™*Ñ$ÒR¨¯©Ð8RÑ)Rð ð Ðr:   )rÕ   rC  r  ry  r_  r`  c                óÐ  ‡ ‡— t        ‰ d«       t        j                  ‰ j                  ‰j                  k(  ˆ ˆfd„«       t	        ‰ ‰«      }|r‰j                  d«      n‰}	t        ‰ |	|d«       t        |	‰ «      \  }
}t        j                  |xs | d„ «       |r|
d d n|
}t        j                  |t        || «      ‰j                  ‰j                  ¬«      }‰ j                  }t        j                  |t        |d«      ‰ j                  ‰ j                  ¬«      }‰ j                  |d d t        j                  ¬«      }‰ j                  |d d t        j                  ¬«      }||||f}||||f}t        d	„ |D «       «      rbt        ||«      D ]S  \  }}t!        ||j                  «       |j#                  |j                  |j%                  «       «       t'        ||d¬
«       ŒU |S )Nzlinalg.solvec                  ó>   •— d‰ j                   › d‰j                   › d�S )NzKlinalg.solve: Expected A and B to have the same dtype, but found A of type r€  rh  r‘   )rö  r  s   €€r8   re   z"_linalg_solve_ex.<locals>.<lambda>Ï  s%   ø€ ØYØ�w‰wˆi� q§w¡w i¨xð9ð r:   rœ   c                   ó   — 	 y)Nz•linalg.solve: Vector broadcasting of the left hand side is not supported for left=False. In this case linalg.solve is equivalent to B / A.squeeze(-1)r4   r4   r:   r8   re   z"_linalg_solve_ex.<locals>.<lambda>Ú  s   € ðKð r:   rW  Fr‘   rò  c              3   ó$   K  — | ]  }|d u–— Œ
 y ­wrD   r4   )rr   r=   s     r8   rt   z#_linalg_solve_ex.<locals>.<genexpr>ñ  s   è ø€ Ò
&˜Qˆ1�DŒ=Ñ
&ùs   ‚)Ú	copy_fromÚcopy_toÚexact_dtype)r®   rV   rg   r^   r·  Ú	unsqueezer  rn  rX  r   rƒ   r™   r˜   ru  rB  rC  r!   rË   rÆ   r#   )rö  r  rÕ   rC  r  ry  r_  r`  r¶  ÚB_ÚB_broad_shaperR   Úresult_shapeÚresult_r™   ÚLU_Úpivots_Úinfo_rÍ   Úresr�  Úos   ``                    r8   Ú_linalg_solve_exrÉ  À  s¶  ù€ ô ˜1˜nÔ-Ü	‡L�LØ	�‰�1—7‘7Ñô	
ôô -¨Q°Ó2€KÙ'ˆ�‰�RŒ¨Q€BÜ�a˜˜T >Ô2Ü3°B¸Ó:Ñ€M�1Ü	‡L�LØÒ�K�ñ	
ôñ *5�=  "Ñ%¸-€LÜ×!Ñ!ØÜ*¨<¸T¸ÓBØ�g‰gØ�x‰xô	€Gð �G‰G€EÜ
×
Ñ
ØÜ*¨5°%Ó8Ø�g‰gØ�x‰xô	€Cð �k‰k˜%  ˜*¬E¯K©KˆkÓ8€GØ�K‰K˜˜c˜r˜
¬%¯+©+ˆKÓ6€EØ�2�v˜tÐ
$€CØ�C˜ %Ð
(€CÜ
Ñ
& #Ô
&Ô&Ü˜˜S“Mò 	F‰DˆAˆqä˜a §¡Ô)à�M‰M˜!Ÿ'™' 1§8¡8£:Ô.Ü Q°¸uÖEð	Fð €Jr:   )rÕ   ÚunitriangularrÍ   rÊ  rÍ   c                óö  — |€| j                  dg«      }t        |t        «      st        dt	        |«      › �«      ‚t        | ||d«       t        || d «      \  }}|j                  dd«      j                  «       xr |j                  «       }|rt        ||j                  «      }|S t        ||j                  «      r=|j                  |j                  dd«      j                  «       |j                  dd«       |S )Nr   zout must be TensorLike, got zlinalg.solve_triangularrò  rœ   )r˜   rp   r   rŠ   rz   r  r:  r0  r–  Úis_conjr!   r™   r"   rÊ   Ú
transpose_)	rö  r  r  rÕ   rÊ  rÍ   rÀ  ÚA_Úavoid_copy_As	            r8   Úlinalg_solve_triangular_metarÐ  û  sÙ   € ð €{Ø�k‰k˜1˜#ÓˆÜ�cœ:Ô&ÜÐ;¼DÀ»I¸;ÐGÓHÐHÜ�a˜˜DÐ";Ô<Ü.¨q°!°TÓ:�F€BˆØ—<‘<  BÓ'×5Ñ5Ó7ÒH¸B¿J¹J»L€LÙÜ  R§X¡XÓ.ˆð €Jô    R§X¡XÔ.Ø�K‰K˜Ÿ™ R¨Ó,×2Ñ2Ô3Ø�N‰N˜2˜rÔ"Ø€Jr:   ÚXÚM)r¾  r0  c                 óø  ‡ ‡— t        j                  ‰ j                  dk\  ˆ fd„«       t        j                  ‰j                  dk\  ˆfd„«       t        ‰ ‰d«       ‰j                  t         j
                  k(  rƒt        ‰ ‰«      \  }}t        j                  |t        |d¬«      ‰ j                  ‰ j                  ¬«      }t        j                  |t        |d¬«      ‰j                  ‰j                  ¬«      }||fS ‰j                  t         j                  k(  s‰j                  t         j                  k(  r+t        j                  ‰ «      }‰ j                  dg«      }||fS t        j                  dd	„ «       fS )
Nrý   c                  ó$   •— d‰ j                   › d�S )NzMtorch.triangular_solve: Expected b to have at least 2 dimensions, but it has r6  r7  r±   s   €r8   re   z'triangular_solve_meta.<locals>.<lambda>!  s   ø€ ðØŸ)™)˜Ð$7ð9ð r:   c                  ó$   •— d‰ j                   › d�S )NzMtorch.triangular_solve: Expected A to have at least 2 dimensions, but it has r6  r7  rþ  s   €r8   re   z'triangular_solve_meta.<locals>.<lambda>(  rf  r:   Útriangular_solveFr!  rW  r   c                   ó   — y)Nz+triangular_solve: Got an unexpected layout.r4   r4   r:   r8   re   z'triangular_solve_meta.<locals>.<lambda>B  rm   r:   )rV   rg   r¥   r   r‚   Ústridedrn  rX  r   r^   rƒ   Ú
sparse_csrÚ
sparse_bsrr¯   r˜   )	rš   rö  r  r0  rÊ  Úself_broadcast_sizeÚA_broadcast_sizeÚsolutionÚcloned_coefficients	   ``       r8   Útriangular_solve_metarß    sL  ù€ ô 
‡L�LØ�	‰	�Q‰ó	
ôô 
‡L�LØ	�‰�!‰ó	
ôô ˜4 Ð$6Ô7à‡x�x”5—=‘=Ò Ü0LÈTÐSTÓ0UÑ-ÐÐ-Ü×&Ñ&Ø$Ü.Ð/BÈeÔTØ—*‘*Ø—;‘;ô	
ˆô #×0Ñ0Ø!Ü.Ð/?È5ÔQØ—'‘'Ø—8‘8ô	
Ðð Ð'Ð'Ð'ð 
�‰”U×%Ñ%Ò	%¨¯©´U×5EÑ5EÒ)EÜ×#Ñ# DÓ)ˆØ!Ÿ^™^¨Q¨CÓ0Ðð Ð'Ð'Ð'ô 	�‰�UÑQÔRØÐ'Ð'Ð'r:   c                 ól  — t        | d«       t        | d«       | j                  | j                  d d «      }| j                  | j                  «      }|j	                  | j                  t        | j                  d¬«      «       | j                  | j                  d d t        j                  ¬«      }|||fS )Nz
linalg.detrò  Fr!  rœ   r‘   r\  )rö  Údetry  r_  s       r8   Ú_linalg_det_metarâ  G  sŠ   € ä�a˜Ô&Ü˜1˜lÔ+à
�+‰+�a—g‘g˜c˜r�lÓ
#€Cà	
�‰�Q—W‘WÓ	€BØ‡N�N�1—7‘7Ô7¸¿¹È5ÔQÔRà�[‰[˜Ÿ™  "˜¬U¯[©[ˆ[Ó9€FØ��Fˆ?Ðr:   c                 ó0  ‡ ‡‡‡‡‡— t        j                  ‰ j                  dk\  d„ «       t        j                  ‰j                  dk\  d„ «       |rdndŠt        j                  ‰j                  ‰   ‰j                  d   k\  ˆfd„«       t        j                  ‰j                  ‰   ‰ j                  d   k(  ˆfd„«       t        j                  ‰j                  d   ‰ j                  d   k  d„ «       t        j                  ‰ j                  ‰j                  z
  d	k(  ˆ ˆfd
„«       t        j                  ‰ j                  ‰j                  k(  ˆ ˆfd„«       ‰ j                  dkD  re‰ j                  d d }‰j                  d d Št        j                  ‰|k(  ˆfd„«       ‰j                  d d Št        j                  ‰|k(  ˆfd„«       t        j                  ‰j                  ‰ j                  k(  ˆ ˆfd„«       t        j                  ‰j                  ‰ j                  k(  ˆ ˆfd„«       t        d‰‰ d«       t        d‰‰ d«       t        j                  ‰j                  t        ‰j                  d¬«      ‰j                  ‰j                  ¬«      S )Nrý   c                   ó   — y)Nz3torch.ormqr: input must have at least 2 dimensions.r4   r4   r:   r8   re   zormqr.<locals>.<lambda>_  rm   r:   c                   ó   — y)Nz3torch.ormqr: other must have at least 2 dimensions.r4   r4   r:   r8   re   zormqr.<locals>.<lambda>b  rm   r:   rò  rœ   c                  ó   •— d‰ › d�S )Nútorch.ormqr: other.shape[z0] must be greater than or equal to tau.shape[-1]r4   ©Úleft_size_conditions   €r8   re   zormqr.<locals>.<lambda>h  s   ø€ Ð+Ð,?Ð+@Ð@pÐq€ r:   c                  ó   •— d‰ › d�S )Nrç  z"] must be equal to input.shape[-2]r4   rè  s   €r8   re   zormqr.<locals>.<lambda>l  s   ø€ Ð+Ð,?Ð+@Ð@bÐc€ r:   c                   ó   — y)NzHtorch.ormqr: tau.shape[-1] must be less than or equal to input.shape[-1]r4   r4   r:   r8   re   zormqr.<locals>.<lambda>q  rm   r:   r3   c                  ó<   •— d‰j                   › d‰ j                   › �S )Nz[torch.ormqr: Expected tau to have one dimension less than input, but got tau.ndim equal to rO  r7  rP  s   €€r8   re   zormqr.<locals>.<lambda>v  rQ  r:   c                  ó<   •— d‰j                   › d‰ j                   › �S )Nzhtorch.ormqr: Expected other to have the same number of dimensions as input, but got other.ndim equal to rO  r7  ©rØ   r¤   s   €€r8   re   zormqr.<locals>.<lambda>}  s+   ø€ ð+Ø+0¯:©:¨,Ð6RÐSX×S]ÑS]ÐR^ð`ð r:   c                  ó   •— d‰ › �S )NzWtorch.ormqr: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r4   rS  s   €r8   re   zormqr.<locals>.<lambda>ˆ  rU  r:   c                  ó   •— d‰ › �S )NzYtorch.ormqr: Expected batch dimensions of other to be equal to input.shape[:-2], but got r4   )Úactual_batch_other_shapes   €r8   re   zormqr.<locals>.<lambda>‘  s   ø€ ð6Ø6NÐ5OðQð r:   c                  ó<   •— d‰ j                   › d‰j                   › �S )NzPtorch.ormqr: Expected input and tau to have the same dtype, but input has dtype z and tau has dtype r‘   rP  s   €€r8   re   zormqr.<locals>.<lambda>™  s'   ø€ ð#Ø#(§;¡; -Ð/BÀ3Ç9Á9À+ðOð r:   c                  ó<   •— d‰ j                   › d‰j                   › �S )NzRtorch.ormqr: Expected input and other to have the same dtype, but input has dtype z and other has dtype r‘   rî  s   €€r8   re   zormqr.<locals>.<lambda>   s'   ø€ ð#Ø#(§;¡; -Ð/DÀUÇ[Á[ÀMðSð r:   ztorch.ormqrrI  r¤   Fr!  rW  )	rV   rg   r¥   r™   r^   r  rX  r   rƒ   )	rØ   rI  r¤   rÕ   r0  Úexpected_batch_shaperñ  rT  ré  s	   ```   @@@r8   Úormqrrõ  U  s  ý€ ô 
‡L�LØ�
‰
�a‰ÑVôô 
‡L�LØ�
‰
�a‰ÑVôñ !%™"¨"ÐÜ	‡L�LØ�‰Ð'Ñ(¨C¯I©I°b©MÑ9Ûqôô 
‡L�LØ�‰Ð'Ñ(¨E¯K©K¸©OÑ;Ûcôô
 
‡L�LØ�	‰	�"‰˜Ÿ™ R™Ñ(ÙZôô
 
‡L�LØ�
‰
�S—X‘XÑ Ñ"ô	
ôô 
‡L�LØ�
‰
�e—j‘jÑ ô	
ôð ‡z�z�A‚~Ø$Ÿ{™{¨3¨BÐ/ÐØ!$§¡¨3¨B ÐÜ�‰Ø"Ð&:Ñ:óô	
ð $)§;¡;¨s°Ð#3Ð Ü�‰Ø$Ð(<Ñ<óô	
ô 
‡L�LØ�	‰	�U—[‘[Ñ ô	
ôô 
‡L�LØ�‰�u—{‘{Ñ"ô	
ôô �M 3¨¨uÔ5Ü�M 5¨%°Ô9ä×ÑØ�[‰[Ü*¨5¯;©;À%ÔHØ�k‰kØ�|‰|ô	ð r:   c                ór  ‡ ‡‡— t        j                  t        ‰«      d‰z  k(  ˆˆfd„«       ‰ j                  }|‰dz   k(  }|}| }|r*t	        d|«      D ]  }|xr ‰ j                  |«      dk7  }Œ n(t	        |«      D ]  }|xr ‰ j                  |«      dk7  }Œ t        j                  |xs |ˆˆ fd„«       y )Nrý   c                  ó,   •— dd‰ z  › dt        ‰«      › �S )Nzpadding size is expected to be rý   z, but got: ©rÃ   )r†   Úpaddings   €€r8   re   z,_padding_check_valid_input.<locals>.<lambda>´  s   ø€ Ð1°!°c±'°¸+ÄcÈ'ÃlÀ^ÐT€ r:   r3   r   c                  ó:   •— d‰ dz   › d‰ dz   › d‰j                   › �S )Nú	Expected r3   zD or rý   zcD (batch mode) tensor with possibly 0 batch size and other non-zero dimensions for input, but got: r$  )r†   rØ   s   €€r8   re   z,_padding_check_valid_input.<locals>.<lambda>É  s2   ø€ Ø˜˜a™�y  c¨A¡g Yð /AØAFÇÁÀðOð r:   )rV   rg   rÃ   r¥   rÅ   r£   )rØ   rù  r†   Ú	input_dimÚis_batch_modeÚvalid_batch_modeÚvalid_non_batch_moderÔ   s   ```     r8   Ú_padding_check_valid_inputr   ±  sÉ   ú€ Ü	‡L�LÜˆG‹˜˜C™ÑÜTôð
 —
‘
€Ià #¨¡'Ñ*€Mà$ÐØ,Ð,Ðáä�q˜)Ó$ò 	GˆAØ/ÒF°E·J±J¸q³MÀQÑ4FÑñ	Gô �yÓ!ò 	OˆAØ#7Ò#N¸E¿J¹JÀq»MÈQÑ<NÑ ð	Oô 
‡L�LØÒ0Ð0ô	
õr:   c                óÞ  ‡ ‡‡‡‡	‡
— d}dŠd}‰ j                   dk(  r‰ j                  d«      }‰dz  Š|dz  }t        ‰ |d¬«       |\  Š	Š
‰ j                  |«      }‰ j                  ‰«      Š‰‰	z   ‰
z   Š|r&t        j                  ‰	‰k  xr ‰
‰k  ˆˆ ˆ	ˆ
fd„«       t        j                  ‰dk\  ˆˆfd„«       ‰ j                   dk(  r‰ j                  |‰f«      S ‰ j                  ||‰f«      S )Nr   r3   r0   r�   c                  ó4   •— d‰› d‰› d‰ › d‰j                   › �S ©NzcArgument #4: Padding size should be less than the corresponding input dimension, but got: padding (rx   ú) at dimension ú
 of input r$  ©Údim_wrØ   Úpad_lÚpad_rs   €€€€r8   re   z_pad1d_common.<locals>.<lambda>å  ó2   ø€ ð%Ø%* G¨2¨e¨W°OÀEÀ7È*ÐUZ×U`ÑU`ÐTaðcð r:   c                  ó   •— d‰ › d‰› �S )Nz
input (W: z%) is too small. Calculated output W: r4   )Úinput_wÚoutput_ws   €€r8   re   z_pad1d_common.<locals>.<lambda>í  s   ø€ �*˜W˜IÐ%JÈ8È*ÐU€ r:   rý   )r¥   r£   r   rV   rg   r˜   )rØ   rù  Úis_reflectionÚ	dim_planeÚnbatchÚnplaner  r  r  r  r	  s   `     @@@@@r8   Ú_pad1d_commonr  Ð  sî   ý€ Ø€IØ€EØ€Fà‡z�z�Q‚Ø—‘˜A“ˆØ�‰
ˆØ�Q‰ˆ	ä˜u g°1Õ5à�L€Eˆ5à�Z‰Z˜	Ó"€FØ�j‰j˜Ó€GØ˜‰ Ñ&€HáÜ�‰Ø�G‰OÒ/ ¨¡öô	
ô 
‡L�LØ�A‰ÜUôð
 ‡z�z�Q‚Ø�‰ ¨Ð1Ó2Ð2à�‰ ¨°Ð9Ó:Ð:r:   c                 ó   — t        | |d¬«      S ©NT©r  )r  ©rØ   rù  s     r8   Úmeta_reflection_pad1dr  ö  ó   € ô ˜ °tÔ<Ð<r:   c                 óˆ   ‡ — t        j                  ‰ j                  t         j                  k7  ˆ fd„«       t	        ‰ |d¬«      S )Nc                  ó@   •— d‰ j                   j                  «       › d�S )Nz)"replication_pad1d" not implemented for 'ú'©r^   Ú__str__©rØ   s   €r8   re   z(meta_replication_pad1d.<locals>.<lambda>  ó    ø€ Ð=¸e¿k¹k×>QÑ>QÓ>SÐ=TÐTUÐX€ r:   Fr  )rV   rg   r^   Úboolr  r  s   ` r8   Úmeta_replication_pad1dr!  ü  ó5   ø€ ô 
‡L�LØ�‰”u—z‘zÑ!ÛXôô ˜ °uÔ=Ð=r:   c                ó   ‡ ‡‡‡‡‡— dŠ|s#t        j                  t        |«      dk(  d„ «       ‰j                  dk(  r‰dz  Š|\  ŠŠ‰j	                  ‰«      }|‰z   ‰z   Š|r&t        j                  ‰|k  xr ‰|k  ˆˆˆˆfd„«       t        j                  ‰‰ j	                  ‰«      k(  ˆˆ ˆfd„«       ‰j                  ‰j                  «      S )Nr3   rý   c                   ó   — y)Nz padding size is expected to be 2r4   r4   r:   r8   re   z(_pad1d_backward_common.<locals>.<lambda>	  rm   r:   r0   c                  ó4   •— d‰› d‰› d‰ › d‰j                   › �S r  r$  r  s   €€€€r8   re   z(_pad1d_backward_common.<locals>.<lambda>  r
  r:   c                  ó2   •— d‰› d‰j                  ‰ «      › �S ©Nz(grad_output width unexpected. Expected: ú, Got: r¢   ©r  Úgrad_outputr  s   €€€r8   re   z(_pad1d_backward_common.<locals>.<lambda>  ó"   ø€ Ð:¸8¸*ÀGÈK×L\ÑL\Ð]bÓLcÐKdÐe€ r:   )rV   rg   rÃ   r¥   r£   r˜   r™   )	r*  rØ   rù  r  r  r  r  r  r	  s	   ``   @@@@r8   Ú_pad1d_backward_commonr,    s¹   ý€ Ø€EÙÜ�‰”S˜“\ QÑ&Ñ(RÔSà‡z�z�Q‚Ø�‰
ˆà�L€Eˆ5à�j‰j˜Ó€GØ˜‰ Ñ&€HáÜ�‰Ø�G‰OÒ/ ¨¡öô	
ô 
‡L�LØ�K×$Ñ$ UÓ+Ñ+Ýeôð
 �?‰?˜5Ÿ;™;Ó'Ð'r:   Ú
grad_inputc                 ó    — t        | ||d¬«      S r  ©r,  ©r*  rØ   rù  s      r8   Úmeta_reflection_pad1d_backwardr1  $  s   € ô " +¨u°gÈTÔRÐRr:   c                 ó    — t        | ||d¬«      S )NFr  r/  r0  s      r8   Úmeta_replication_pad1d_backwardr3  *  s   € ô " +¨u°gÈUÔSÐSr:   c                ó’  ‡ ‡‡‡	‡
‡‡‡‡‡‡— dŠdŠd}d}t        ‰ |d¬«       ‰ j                  }|dk(  r ‰ j                  d«      }‰dz  Š‰dz  Š|dz  }|\  ŠŠŠŠ‰ j                  |«      }‰ j                  ‰«      Š	‰ j                  ‰«      Š
‰	‰z   ‰z   Š‰
‰z   ‰z   Š|rLt        j                  ‰‰
k  xr ‰‰
k  ˆˆ ˆˆfd„«       t        j                  ‰‰	k  xr ‰‰	k  ˆˆ ˆˆfd„«       t        j                  ‰dk\  xs ‰dk\  ˆ	ˆ
ˆˆfd„«       ‰ j                  d	k(  r‰ j                  |‰‰f«      S ‰ j                  ||‰‰f«      S )
Nrý   r3   r   r�   é   c                  ó4   •— d‰› d‰› d‰ › d‰j                   › �S r  r$  r  s   €€€€r8   re   z_pad2d_common.<locals>.<lambda>J  r
  r:   c                  ó4   •— d‰› d‰› d‰ › d‰j                   › �S ©NzcArgument #6: Padding size should be less than the corresponding input dimension, but got: padding (rx   r  r  r$  ©Údim_hrØ   Úpad_bÚpad_ts   €€€€r8   re   z_pad2d_common.<locals>.<lambda>Q  r
  r:   c                  ó    •— d‰ › d‰› d‰› d‰› �S )Nz
input (H: ú W: z%) is too small. Calculated output H: r4   )Úinput_hr  Úoutput_hr  s   €€€€r8   re   z_pad2d_common.<locals>.<lambda>Y  s*   ø€ Ø˜˜	  g Yð /$Ø$, :¨T°(°ð=ð r:   r0   ©r   r¥   r£   rV   rg   r˜   )rØ   rù  r  Ú
dim_slicesr  r¥   r  r:  r  r?  r  r@  r  r;  r  r	  r<  s   `      @@@@@@@@@@r8   Ú_pad2d_commonrC  0  sU  ÿú€ Ø€EØ€EØ€JØ€Fä˜u g°1Õ5à�:‰:€DØˆq‚yØ—‘˜A“ˆØ�‰
ˆØ�‰
ˆØ�a‰ˆ
à!(Ñ€Eˆ5�%˜à�Z‰Z˜
Ó#€FØ�j‰j˜Ó€GØ�j‰j˜Ó€GØ˜‰ Ñ&€HØ˜‰ Ñ&€HáÜ�‰Ø�G‰OÒ/ ¨¡öô	
ô 	�‰Ø�G‰OÒ/ ¨¡öô	
ô 
‡L�LØ�A‰Ò&˜ Q™ö	
ôð ‡z�z�Q‚Ø�‰ ¨°(Ð;Ó<Ð<à�‰ ¨°¸(ÐCÓDÐDr:   c                 ó   — t        | |d¬«      S r  )rC  r  s     r8   Úmeta_reflection_pad2drE  e  r  r:   c                 óˆ   ‡ — t        j                  ‰ j                  t         j                  k7  ˆ fd„«       t	        ‰ |d¬«      S )Nc                  ó@   •— d‰ j                   j                  «       › d�S )Nz)"replication_pad2d" not implemented for 'r  r  r  s   €r8   re   z(meta_replication_pad2d.<locals>.<lambda>p  r  r:   Fr  )rV   rg   r^   r   rC  r  s   ` r8   Úmeta_replication_pad2drH  k  r"  r:   c                 ó^   — t        j                  |«      }t        j                  |«      }||fS rD   ©rV   r¯   )Úgrad_wÚsaved_vÚsaved_gÚsaved_normsr†   Úgrad_vÚgrad_gs          r8   Úmeta_weight_norm_backwardrQ  u  s.   € ô ×Ñ˜gÓ&€FÜ×Ñ˜gÓ&€FØ�6ˆ>Ðr:   c                 ó   ‡ ‡‡‡‡— dŠdŠd}|j                   }|j                  «       dk(  r‰dz  Š‰dz  Š|dz  }|\  }}}}|‰   }	|‰   }
|	|z   |z   Š|
|z   |z   Št        j                  ‰‰ j	                  ‰«      k(  ˆˆ ˆfd„«       t        j                  ‰‰ j	                  ‰«      k(  ˆˆ ˆfd„«       |j                  |j                   «      S )Nrý   r3   r   r5  c                  ó2   •— d‰› d‰j                  ‰ «      › �S r'  r¢   r)  s   €€€r8   re   z%meta_pad2d_backward.<locals>.<lambda>›  r+  r:   c                  ó2   •— d‰› d‰j                  ‰ «      › �S ©Nz)grad_output height unexpected. Expected: r(  r¢   ©r:  r*  r@  s   €€€r8   re   z%meta_pad2d_backward.<locals>.<lambda>Ÿ  ó"   ø€ Ð;¸H¸:ÀWÈ[×M]ÑM]Ð^cÓMdÐLeÐf€ r:   )r™   r†   rV   rg   r£   r˜   )r*  rš   rù  r  rd   r  r	  r<  r;  r?  r  r:  r  r@  r  s   `          @@@@r8   Úmeta_pad2d_backwardrX  ~  sÞ   ü€ ð €EØ€EØ€Ià—‘€JØ‡x�xƒz�Q‚Ø�‰
ˆØ�‰
ˆØ�Q‰ˆ	à!(Ñ€Eˆ5�%˜à˜Ñ€GØ˜Ñ€GØ˜‰ Ñ&€HØ˜‰ Ñ&€Hä	‡L�LØ�K×$Ñ$ UÓ+Ñ+Ýeôô 
‡L�LØ�K×$Ñ$ UÓ+Ñ+Ýfôð �>‰>˜$Ÿ*™*Ó%Ð%r:   c          	      ó$  ‡ ‡‡‡	‡
‡‡‡‡‡‡‡‡‡‡‡— dŠ	dŠdŠd}t        ‰ |d¬«       ‰ j                  dk(  }|r%‰ j                  d«      }‰	dz  Š	‰dz  Š‰dz  Š|dz  }|\  ŠŠŠŠŠŠ‰ j                  |«      }‰ j                  ‰«      Š
‰ j                  ‰«      Š‰ j                  ‰	«      Š‰
‰z   ‰z   Š‰‰z   ‰z   Š‰‰z   ‰z   Š|rrt        j                  ‰‰k  xr ‰‰k  ˆ	ˆ ˆˆfd„«       t        j                  ‰‰k  xr ‰‰k  ˆˆ ˆˆfd„«       t        j                  ‰‰
k  xr ‰‰
k  ˆˆ ˆˆfd	„«       t        j                  ‰dk\  xs ‰dk\  xs ‰dk\  ˆ
ˆˆˆˆˆfd
„«       |r‰ j                  |‰‰‰f«      S ‰ j                  |‰‰‰f«      S )Nr0   rý   r3   r   r�   é   c                  ó4   •— d‰› d‰› d‰ › d‰j                   › �S r  r$  r  s   €€€€r8   re   z_pad3d_common.<locals>.<lambda>Á  r
  r:   c                  ó4   •— d‰› d‰› d‰ › d‰j                   › �S r8  r$  r9  s   €€€€r8   re   z_pad3d_common.<locals>.<lambda>È  r
  r:   c                  ó4   •— d‰› d‰› d‰ › d‰j                   › �S )NzcArgument #8: Padding size should be less than the corresponding input dimension, but got: padding (rx   r  r  r$  )Údim_drØ   Úpad_bkÚpad_fs   €€€€r8   re   z_pad3d_common.<locals>.<lambda>Ï  s2   ø€ ð%Ø%* G¨2¨f¨X°_ÀUÀGÈ:ÐV[×VaÑVaÐUbðdð r:   c                  ó,   •— d‰ › d‰› d‰› d‰› d‰› d‰› �S )Nz
input (D: ú H: r>  z%) is too small. Calculated output D: r4   )Úinput_dr?  r  Úoutput_dr@  r  s   €€€€€€r8   re   z_pad3d_common.<locals>.<lambda>×  s7   ø€ Ø˜˜	  g Y¨d°7°)ð <$Ø$, :¨T°(°¸4À¸zðKð r:   rA  )rØ   rù  r  r  Ú
batch_moder  r  r^  r:  r  rc  r?  r  rd  r@  r  r;  r_  r`  r  r	  r<  s   `      @@@@@@@@@@@@@@@r8   Ú_pad3d_commonrf  ¤  s¬  ÿÿ€ Ø€EØ€EØ€EØ€Iä˜u g°1Õ5à—‘˜q‘€JÙØ—‘˜A“ˆØ�‰
ˆØ�‰
ˆØ�‰
ˆØ�Q‰ˆ	à07Ñ-€Eˆ5�%˜  và�Z‰Z˜	Ó"€FØ�j‰j˜Ó€GØ�j‰j˜Ó€GØ�j‰j˜Ó€GØ˜‰ Ñ'€HØ˜‰ Ñ&€HØ˜‰ Ñ&€HáÜ�‰Ø�G‰OÒ/ ¨¡öô	
ô 	�‰Ø�G‰OÒ/ ¨¡öô	
ô 	�‰Ø�G‰OÒ0 ¨Ñ 0öô	
ô 
‡L�LØ�A‰Ò7˜ Q™Ò7¨(°a©-÷	
ð 	
ôñ Ø�‰ ¨°¸(ÀHÐMÓNÐNà�‰ ¨°(¸HÐEÓFÐFr:   c                 ó   — t        | |d¬«      S r  )rf  r  s     r8   Úmeta_reflection_pad3drh  ã  r  r:   c                 óˆ   ‡ — t        j                  ‰ j                  t         j                  k7  ˆ fd„«       t	        ‰ |d¬«      S )Nc                  ó@   •— d‰ j                   j                  «       › d�S )Nz)"replication_pad3d" not implemented for 'r  r  r  s   €r8   re   z(meta_replication_pad3d.<locals>.<lambda>î  r  r:   Fr  )rV   rg   r^   r   rf  r  s   ` r8   Úmeta_replication_pad3drk  é  r"  r:   c                 óT  ‡ ‡‡‡‡‡‡— t        j                  t        |«      dk(  d„ «       |j                  dk  rt	        d|j                  › �«      ‚‰ j                  |j                  k7  r%t	        d‰ j                  › d|j                  › �«      ‚dŠdŠdŠ|j                  d	k(  r‰dz  Š‰dz  Š‰dz  Š|\  }}}}}}|j                  ‰«      }	|j                  ‰«      }
|j                  ‰«      }|	|z   |z   Š|
|z   |z   Š||z   |z   Št        j                  ‰‰ j                  ‰«      k(  ˆˆ ˆfd
„«       t        j                  ‰‰ j                  ‰«      k(  ˆˆ ˆfd„«       t        j                  ‰‰ j                  ‰«      k(  ˆˆ ˆfd„«       |j                  |j                  «      S )Né   c                   ó   — y)Nz padding size is expected to be 6r4   r4   r:   r8   re   z%meta_pad3d_backward.<locals>.<lambda>ý  rm   r:   r0   zinput.ndim must be > 3, got z,grad_output.ndim must equal input.ndim, got ú != rý   r3   rZ  c                  ó2   •— d‰› d‰j                  ‰ «      › �S r'  r¢   r)  s   €€€r8   re   z%meta_pad3d_backward.<locals>.<lambda>	  r+  r:   c                  ó2   •— d‰› d‰j                  ‰ «      › �S rU  r¢   rV  s   €€€r8   re   z%meta_pad3d_backward.<locals>.<lambda>	  rW  r:   c                  ó2   •— d‰› d‰j                  ‰ «      › �S )Nz(grad_output depth unexpected. Expected: r(  r¢   )r^  r*  rd  s   €€€r8   re   z%meta_pad3d_backward.<locals>.<lambda>!	  r+  r:   )rV   rg   rÃ   r¥   rŠ   r£   r˜   r™   )r*  rØ   rù  r  r	  r<  r;  r`  r_  rc  r?  r  r^  r:  r  rd  r@  r  s   `           @@@@@@r8   Úmeta_pad3d_backwardrs  ó  s™  þ€ ô 
‡L�L”�W“ Ñ"Ñ$NÔOØ‡z�z�Q‚ÜÐ;¸E¿J¹J¸<ÐHÓIÐIØ×Ñ˜5Ÿ:™:Ò%ÜØ:¸;×;KÑ;KÐ:LÈDÐQV×Q[ÑQ[ÐP\Ð]ó
ð 	
ð €EØ€EØ€Eà‡z�z�Q‚Ø�‰
ˆØ�‰
ˆØ�‰
ˆà07Ñ-€Eˆ5�%˜  và�j‰j˜Ó€GØ�j‰j˜Ó€GØ�j‰j˜Ó€GØ˜‰ Ñ'€HØ˜‰ Ñ&€HØ˜‰ Ñ&€Hä	‡L�LØ�K×$Ñ$ UÓ+Ñ+Ýeôô 
‡L�LØ�K×$Ñ$ UÓ+Ñ+Ýfôô 
‡L�LØ�K×$Ñ$ UÓ+Ñ+Ýeôð
 �?‰?˜5Ÿ;™;Ó'Ð'r:   Úpc                 óJ  — t        j                  | j                  «       d„ «       | j                  d«      }|dk  r0| j	                  dg«      j                  t         j                  ¬«      S | j	                  ||dz
  z  dz  f«      j                  t         j                  ¬«      S )Nc                   ó   — y)Nz(_pdist_forward requires contiguous inputr4   r4   r:   r8   re   z%meta__pdist_forward.<locals>.<lambda>+	  rm   r:   r   r3   r«   rý   )rV   rg   r–  r£   r˜   rY  r?  )rš   rt  r  s      r8   Úmeta__pdist_forwardrw  '	  s”   € ô 
‡L�LØ×ÑÓÑPôð 	�	‰	�!‹€AØˆA‚vØ�~‰~˜q˜cÓ"×%Ñ%´E×4RÑ4RÐ%ÓSÐSà�~‰~˜q A¨¡E™{¨aÑ/Ð1Ó2×5Ñ5Ü×8Ñ8ð 6ó 
ð 	
r:   ÚgradÚpdistc                 óà   — t        j                  |j                  «       d„ «       t        j                  |j                  «       d„ «       t        j                  |t         j                  ¬«      S )Nc                   ó   — y)Nz._pdist_backward requires self to be contiguousr4   r4   r:   r8   re   z&meta__pdist_backward.<locals>.<lambda>:	  rm   r:   c                   ó   — y)Nz/_pdist_backward requires pdist to be contiguousr4   r4   r:   r8   re   z&meta__pdist_backward.<locals>.<lambda>=	  rm   r:   r«   )rV   rg   r–  r¯   r?  )rx  rš   rt  ry  s       r8   Úmeta__pdist_backwardr}  6	  sW   € ô 
‡L�LØ×ÑÓÑVôô 
‡L�LØ×ÑÓÑXôô ×Ñ˜D´×0NÑ0NÔOÐOr:   )rˆ  r‡  c          
      ó0  ‡ ‡‡‡‡‡— ddl m}m} ‰j                  d«      }‰j                  d«      }‰j                  d«      }	 |t	        j
                   |‰ j                  |||	f«      «      «      r‰ j                  |||	f«      Š t	        j                  ‰j                  «       dk(  d„ «       t	        j                  ‰j                  «       dk(  d„ «       t        j                  sGt	        j                  ‰ j                  ‰j                  cxk(  xr ‰j                  k(  nc ˆˆˆ fd„«       ‰j                  }
‰j                  Š|
d   Š|
d   Št	        j                  ‰d   ‰k(  xr ‰d   ‰k(  ˆˆˆfd	„«       ‰ j                  ‰ j                  «       «      S )
Nr   )Úguard_or_trueÚsym_eqr3   rý   r0   c                   ó   — y©Nzbatch1 must be a 3D tensorr4   r4   r:   r8   re   zmeta_baddbmm.<locals>.<lambda>L	  rm   r:   c                   ó   — y©Nzbatch2 must be a 3D tensorr4   r4   r:   r8   re   zmeta_baddbmm.<locals>.<lambda>M	  rm   r:   c                  óV   •— d‰j                   › d‰ j                   › d‰j                   › �S )Nz+Input dtypes must be the same, got: input: z
, batch1: z
, batch2: r‘   )Úbatch1Úbatch2rš   s   €€€r8   re   zmeta_baddbmm.<locals>.<lambda>Q	  s0   ø€ ÐAÀ$Ç*Á*ÀÈZÐX^×XdÑXdÐWeÐeoÐpv×p|Ñp|Ðo}Ð~€ r:   c            	      ó.   •— d‰› d‰› d‰ d   › d‰ d   › d�	S ©Nz@Expected size for first two dimensions of batch2 tensor to be: [rx   z] but got: [r   r3   ú].r4   ©Úbatch2_sizesÚbsÚcontraction_sizes   €€€r8   re   zmeta_baddbmm.<locals>.<lambda>Y	  s:   ø€ ðØˆt�2Ð&Ð' |°LÀ±OÐ3DÀBÀ|ÐTUÁÐFWÐWYð[ð r:   )rV  r  r€  r£   rV   Úsym_notr™   r[  rg   r†   Ú
exp_configÚ&skip_dtype_check_in_meta_registrationsr^   r˜   )rš   r†  r‡  rˆ  r‡  r  r€  Údim1Údim2Údim3Úbatch1_sizesrŒ  r�  rŽ  s   ```        @@@r8   Úmeta_baddbmmr–  B	  s7  ý€ ÷ Là�;‰;�q‹>€DØ�;‰;�q‹>€DØ�;‰;�q‹>€DÙ”U—]‘]¡6¨$¯*©*°t¸TÀ4Ð6HÓ#IÓJÔKØ�{‰{˜D $¨Ð-Ó.ˆÜ	‡L�L�—‘“ Ñ"Ñ$HÔIÜ	‡L�L�—‘“ Ñ"Ñ$HÔIÜ×<Ò<Ü�‰Ø�J‰J˜&Ÿ,™,Ö6¨&¯,©,Ô6Ý~ô	
ð —<‘<€LØ—<‘<€LØ	�a‰€BØ# A‘ÐÜ	‡L�LØ�Q‰˜2ÑÒE ,¨q¡/Ð5EÑ"Eõ	
ôð �>‰>˜$Ÿ)™)›+Ó&Ð&r:   c                óL   — t        j                  | t         j                  ¬«      S r»   rŸ  ©rš   r
  s     r8   Úmeta_bernoullir™  a	  s   € ô ×Ñ˜D´×0GÑ0GÔHÐHr:   c                 ó   — | S rD   r4   ©rš   rt  r
  s      r8   Úmeta_bernoulli_rœ  h	  ó   € à€Kr:   c                 óL   — t        j                  | t         j                  ¬«      S r»   rŸ  r›  s      r8   Úmeta_bernoulli_prŸ  m	  s   € ô ×Ñ˜D´×0GÑ0GÔHÐHr:   c                 ó,   — t        j                  | «      S rD   rJ  r˜  s     r8   Úmeta_poissonr¡  s	  ó   € ô ×Ñ˜DÓ!Ð!r:   c                 óÊ   — t        j                  |
| j                  «       k  d„ «       t        j                  | t         j                  ¬«      }t        j                  | «      |fS )Nc                   ó   — y)NzJError in fused_moving_avg_obs_fake_quant_cpu: ch_axis must be < self.dim()r4   r4   r:   r8   re   z6meta__fused_moving_avg_obs_fq_helper.<locals>.<lambda>‹	  rm   r:   r‘   )rV   rg   r†   r¯   r   )rš   Úobserver_onÚfake_quant_onÚrunning_minÚrunning_maxÚscaleÚ
zero_pointÚaveraging_constÚ	quant_minÚ	quant_maxÚch_axisÚper_row_fake_quantÚsymmetric_quantÚmasks                 r8   Ú$meta__fused_moving_avg_obs_fq_helperr²  y	  sO   € ô  
‡L�LØ�$—(‘(“*ÑÙ\ôô ×Ñ˜D¬¯
©
Ô3€DÜ×Ñ˜TÓ" DÐ)Ð)r:   c                 ó@  ‡‡‡‡— t        j                  | j                  «       dk(  d„ «       t        j                  |j                  «       dk(  d„ «       | j                  \  ŠŠ|j                  \  ŠŠt        j                  ‰‰k(  ˆˆˆˆfd„«       |�gt        j                  || j                  k(  xsA |t         j
                  k(  xr, | j                  t         j                  t         j                  fv d„ «       |€| j                  n|}| j                  ‰‰f|¬«      S )Nrý   c                   ó   — y)Nza must be 2Dr4   r4   r:   r8   re   zmeta_mm.<locals>.<lambda>”	  rm   r:   c                   ó   — y)Nzb must be 2Dr4   r4   r:   r8   re   zmeta_mm.<locals>.<lambda>•	  rm   r:   c            	      ó"   •— d‰› d‰ › d‰› d‰› d�	S )Nz/a and b must have same reduction dim, but got [rx   z] X [rŠ  r4   )ÚM1ÚM2ÚNrr  s   €€€€r8   re   zmeta_mm.<locals>.<lambda>š	  s(   ø€ ÐAÀ!ÀÀBÀrÀdÈ%ÐPRÈtÐSUÐVWÐUXÐXZÐ[€ r:   c                   ó   — y)NzFout_dtype must be the same as input dtype or fp32 for fp16/bf16 inputsr4   r4   r:   r8   re   zmeta_mm.<locals>.<lambda>£	  rm   r:   r‘   )	rV   rg   r†   r™   r^   r’  r“  r”  r˜   )r6   r7   ro  rS   r·  r¸  r¹  rr  s       @@@@r8   Úmeta_mmr»  ‘	  sâ   û€ ô 
‡L�L�—‘“˜A‘Ñ5Ô6Ü	‡L�L�—‘“˜A‘Ñ5Ô6Ø�G‰G�E€A€rØ�G‰G�E€BˆÜ	‡L�LØ
ˆb‰Þ[ôð ÐÜ�‰Ø˜Ÿ™Ñ ò àœUŸ]™]Ñ*ò ?Ø—G‘G¤§¡¬u¯~©~Ð>Ð>á\ô	
ð (Ð/�1—7’7°Y€LØ�;‰;˜˜1�v \ˆ;Ó2Ð2r:   c                 óš   ‡ ‡— |r(t        ˆˆ fd„t        ‰ j                  «      D «       «      S t        j                  ‰ j
                  ‰«      S )Nc              3   óH   •K  — | ]  }|‰vr‰j                   |   nd –— Œ y­wr<  r$  )rr   rÜ   Údimsrš   s     €€r8   rt   z+_compute_reduction_shape.<locals>.<genexpr>«	  s$   øè ø€ ÒU¸q a¨t¡m�T—Z‘Z ’]¸Ó:ÑUùs   ƒ")rf   rÅ   r¥   rO   Úcompute_reduction_output_shaper™   )rš   r¾  r¿  s   `` r8   r½  r½  ©	  s7   ù€ ÙÜÔUÄEÈ$Ï)É)ÓDTÔUÓUÐUä×/Ñ/°·
±
¸DÓAÐAr:   c                 ó  — t        | t        j                  j                  «      r| j                  j
                  S t        | d«      rEt        | j                  d«      r/| j                  j
                  dk7  r| j                  j
                  S y)Nrƒ   rz   r€   ré   )rp   rV   Ú_subclassesÚ
FakeTensorÚfake_devicerz   Úhasattrrƒ   )rb  s    r8   rð   rð   ´	  sg   € Ü�&œ%×+Ñ+×6Ñ6Ô7Ø×!Ñ!×&Ñ&Ð&ä�˜Ô!Ü�F—M‘M 6Ô*Ø�M‰M×Ñ &Ò(à�}‰}×!Ñ!Ð!àr:   Úinput_tensorrÆ   rù  ÚdilationÚis_transposedÚgroupsÚoutput_paddingc                 óê  ‡‡— dt         dt         dt         dt         dt         dt         fd„}dt         dt         dt         dt         dt         dt         dt         fd	„}	|j                  d
d  }
| j                  d
d  Š|r||j                  d   z  }nF|j                  d   }t        j                  |j                  d   |z  | j                  d   k(  d„ «       | j                  d   |gŠt	        |t
        «      r|gt        ‰«      z  }n t        |«      dk(  r|d   gt        ‰«      z  }t	        |t
        «      r|gt        ‰«      z  }n t        |«      dk(  r|d   gt        ‰«      z  }t	        |t
        «      r|gt        ‰«      z  }n t        |«      dk(  r|d   gt        ‰«      z  }d }|rCt	        |t
        «      r|gt        ‰«      z  }n#t        |«      dk(  r|d   gt        ‰«      z  }n|}t        t        ‰«      «      D ]]  }|r/‰j                   |	‰|   ||   ||   |
|   ||   ||   «      «       Œ4‰j                   |‰|   ||   ||   |
|   ||   «      «       Œ_ ddl	m
} ddlm} t	        | |«      r| j                  n| j                  }|j                  dk(  xr t        j                   j"                  d u }|s1t        j                   |‰d
d  D �cg c]  }|dkD  ‘Œ	 c}Ž ˆˆfd„«       ‰S c c}w )NÚlnrt  rÔ   rw  rØ  r@   c                 ó6   — | d|z  z   ||dz
  z  z
  dz
  |z  dz   S )a�  
        Formula to apply to calculate the length of some dimension of the output

        See: https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html

        Args:
            ln: length of the dimension
            p: padding in that dim
            d: dilation in that dim
            k: kernel size in that dim
            s: stride in that dim
        Returns:
            The output length
        rý   r3   r4   )rË  rt  rÔ   rw  rØ  s        r8   Ú_formulaz+calc_conv_nd_return_shape.<locals>._formulaË	  s.   € ð �Q˜‘U‘
˜Q ! a¡%™[Ñ(¨1Ñ,°Ñ2°QÑ6Ð6r:   rE   c                 ó<   — | dz
  |z  d|z  z
  ||dz
  z  z   |z   dz   S )aü  
        Formula to apply to calculate the length of some dimension of the output
        if transposed convolution is used.
        See: https://pytorch.org/docs/stable/generated/torch.nn.ConvTranspose2d.html

        Args:
            ln: length of the dimension
            p: padding in that dim
            d: dilation in that dim
            k: kernel size in that dim
            s: stride in that dim
            op: output padding in that dim

        Returns:
            The output length
        r3   rý   r4   )rË  rt  rÔ   rw  rØ  rE   s         r8   Ú_formula_transposedz6calc_conv_nd_return_shape.<locals>._formula_transposedÜ	  s2   € ð" �Q‘˜!‰|˜a !™eÑ# a¨1¨q©5¡kÑ1°BÑ6¸Ñ:Ð:r:   rý   r3   r   c                   ó   — y)NzInvalid channel dimensionsr4   r4   r:   r8   re   z+calc_conv_nd_return_shape.<locals>.<lambda>÷	  rm   r:   )rÂ  )Úsym_orré   c                  ó.   •— dt        ‰ «      › d‰dd  › d�S )NzGiven input size per channel: z&. Calculated output size per channel: rý   z. Output size is too small)rÄ   )r¾  Ú	ret_shapes   €€r8   re   z+calc_conv_nd_return_shape.<locals>.<lambda>@
  s,   ø€ Ð4´T¸$³Z°Lð A3Ø3<¸Q¸R°=°/ð B'ð(€ r:   )rã   r™   rV   rg   rp   r   rÃ   rÅ   ÚappendÚtorch._subclasses.fake_tensorrÂ  rV  rÑ  rÃ  rƒ   rz   ÚversionÚhip)rÅ  rk  rÆ   rù  rÆ  rÇ  rÈ  rÉ  rÍ  rÏ  Úkernel_sizeÚout_channelsÚoutput_padding_listrÜ   rÂ  rÑ  rƒ   Úis_cudnnr=   r¾  rÓ  s                      @@r8   Úcalc_conv_nd_return_shaperÜ  Á	  s  ù€ ð7”Sð 7œSð 7¤Sð 7¬Sð 7´Sð 7¼Só 7ð";¤ð ;¬ð ;´ð ;¼ð ;Äð ;Ìð ;ÔQTó ;ð& —,‘,˜q˜rÐ"€KØ×Ñ˜a˜bÐ!€DÙØ §¡¨Q¡Ñ/‰à—|‘| A‘ˆÜ�‰Ø�L‰L˜‰O˜fÑ$¨×(:Ñ(:¸1Ñ(=Ñ=Ù0ô	
ð
 ×#Ñ# AÑ&¨Ð5€IÜ�&œ'Ô"à�œC ›IÑ%‰Ü	ˆV‹˜Ò	Ø˜‘)�œs 4›yÑ(ˆä�'œ7Ô#à�)œc $›iÑ'‰Ü	ˆW‹˜Ò	Ø˜1‘:�,¤ T£Ñ*ˆä�(œGÔ$à�:¤ D£	Ñ)‰Ü	ˆX‹˜!Ò	Ø˜Q‘K�=¤3 t£9Ñ,ˆà,0ÐÙÜ�n¤gÔ.à#1Ð"2´S¸³YÑ">ÑÜ�Ó  AÒ%Ø#1°!Ñ#4Ð"5¼¸D»	Ñ"AÑà"0Ðä”3�t“9Óò ˆáØ×ÑÙ#Ø˜‘Gà˜A‘Jð ˜Q‘KØ ‘Nà˜1‘IØ'¨Ñ*óõð ×Ñá˜˜a™ '¨!¡*¨h°q©k¸;Àq¹>È6ÐRSÉ9ÓUõð%õ6 9Ý<ô �l JÔ/ð 	× Ò à× Ñ ð ð �{‰{˜fÑ$ÒB¬¯©×):Ñ):¸dÐ)B€HÙÜ�‰Ù I¨a¨b MÖ2˜q�Q˜“UÒ2Ð3ô(ô	
ð Ðùò 3s   ËK0c                 ób   — t         j                  j                  | «      t         j                  k(  S rD   ©rV   Ú_prims_commonr   Úchannels_last©Útens    r8   Úis_channels_lastrã  H
  s$   € Ü×Ñ×4Ñ4°SÓ9¼U×=PÑ=PÑPÐPr:   Úrunning_meanÚrunning_varÚtrainingÚexponential_average_factorÚepsilonc                 ór  ‡ — ‰ j                   }|�|j                   n|j                   }	|�|j                   n|j                   }
ˆ fd„}‰ j                  |«      j                   |«       ¬«      }|r#‰ j                  |	«      }‰ j                  |
«      }n"‰ j                  d«      }‰ j                  d«      }|||fS )Nc                  óº   •— t        ‰ «      rt        j                  S ‰ j                  t        j                  ¬«      rt        j                  S t        j                  S r»   )rã  rV   rà  r–  r°   )rÅ  s   €r8   Úpick_memory_formatz2meta_miopen_batch_norm.<locals>.pick_memory_format_
  sF   ø€ Ü˜LÔ)Ü×&Ñ&Ð&Ø×%Ñ%´E×4KÑ4KÐ%ÔLÜ×*Ñ*Ð*Ü×&Ñ&Ð&r:   r«   ©r   )r™   r˜   rY  )rÅ  rk  rm  rä  rå  ræ  rç  rè  r¨   Úsave_mean_shapeÚsave_var_shaperë  rÍ   Ú	save_meanÚsave_vars   `              r8   Úmeta_miopen_batch_normrñ  L
  s·   ø€ ð ×"Ñ"€Ið -9Ð,D�l×(Ò(È&Ï,É,€OØ*5Ð*A�[×&Ò&ÀvÇ|Á|€Nô'ð ×
 Ñ
  Ó
+×
.Ñ
.Ñ=OÓ=QÐ
.Ó
R€CáØ ×*Ñ*¨?Ó;ˆ	Ø×)Ñ)¨.Ó9‰à ×*Ñ*¨4Ó0ˆ	Ø×)Ñ)¨$Ó/ˆà�	˜8Ð#Ð#r:   c	           
      ó¦   — t        | |||||||r|nd «      }	ddlm}
 d}d} |
| j                  |«      dk(  «      rd|	|<   | j	                  |	«      }|S )Nr   ©Úguard_or_falser3   )rÜ  rV  rô  r£   r˜   )rÅ  rk  rm  rÆ   rù  rÆ  rÇ  rÉ  rÈ  Ú	shape_outrô  Úinput_channels_dimÚoutput_channels_dimrÍ   s                 r8   Ú	meta_convrø  r
  sw   € ô *ØØØØØØØÙ'‰¨Tó	€Iõ EàÐØÐÙ�l×'Ñ'Ð(:Ó;¸qÑ@ÔAØ)*ˆ	Ð%Ñ&ð ×
 Ñ
  Ó
+€CØ€Jr:   Úmkldnnc
           
      óÖ   — t        | ||||d|g «      }
| j                  |
«      }t        j                  }| j	                  «       dk(  rt        j
                  }|j                  |¬«      }|S )NFrZ  r«   )rÜ  r˜   rV   rà  r†   Úchannels_last_3drY  )rÅ  rk  rm  rù  rÆ   rÆ  rÈ  ÚattrÚscalarsÚ	algorithmrõ  rÍ   Úout_memory_formats                r8   Úmeta_mkldnn_convolution_defaultr   ž
  sp   € ô .Ø˜& &¨'°8¸UÀFÈBó
ˆ	ð ×$Ñ$ YÓ/ˆÜ!×/Ñ/ÐØ×ÑÓ Ò"Ü %× 6Ñ 6ÐØ�f‰fÐ#4ˆfÓ5ˆØˆ
r:   c                 ób   — | j                  g | j                  d d ¢|j                  d   ‘­«      S ©Nrœ   r   ©r˜   r™   )rÅ  rk  rm  rü  rý  rþ  s         r8   Úmeta_linear_pointwise_defaultr  µ
  s5   € ð ×%Ñ%Ð&Q¨×(:Ñ(:¸3¸BÐ(?Ð&QÀÇÁÈaÁÑ&QÓRÐRr:   rò   c                 ób   — | j                  g | j                  d d ¢|j                  d   ‘­«      S r  r  )rÅ  Úpacked_weightÚorig_weightrm  rÚ   s        r8   Úmeta_mkl_linearr  À
  s:   € à×)Ñ)Ø@�,×$Ñ$ S bÐ)Ð@¨;×+<Ñ+<¸QÑ+?Ñ@óð r:   Úonednnc           
      ó  — t        | ||||	d|
d «      }|€| j                  }|t        j                  t        j                  t        j
                  t        j                  t        j                  fvrt        d|› �«      ‚| j                  ||¬«      }t        |«      dvrt        dt        |«      › d�«      ‚t        j                  t        j                  t        j                  dœt        |«         }|j                  |¬«      }|S )NFúOoutput_dtype must be one of float32, bfloat16, uint8, int8, float8_e4m3fn, got r‘   )r0   r5  rZ  z3Expect output to be 3d/4d/5d for conv1d/2d/3d, got rÔ   r«   )rÜ  r^   rV   r’  r”  Úuint8rt  r•  rŠ   r˜   rÃ   r°   rà  rû  rY  )r=   Úx_scaleÚx_zpÚwÚw_scaleÚw_zprm  rÆ   rù  rÆ  rÈ  Úoutput_scaleÚoutput_zero_pointÚoutput_dtyperü  rý  rþ  rõ  rÍ   Úformats                       r8   Úmeta_qconv_pointwiser  Ê
  s  € ô, .ØØØØØØØØó	
ˆ	ð ÐØŸ7™7ˆLØÜ�M‰MÜ�N‰NÜ�K‰KÜ�J‰JÜ×Ñð 
ñ 
ô !ØaÐbnÐaoÐpóð ð �k‰k˜)¨<ˆkÓ8ˆÜˆy‹> Ñ*Ü ØEÄcÈ)ÃnÐEUÐUVÐWóð ô ×&Ñ&Ü×"Ñ"Ü×%Ñ%ñ
ô ˆi‹.ñ	ˆð
 �f‰f 6ˆfÓ*ˆØˆ
r:   c                 ó.   — |dk7  rt        d|› d�«      ‚|S )NÚsumz#binary_op_name must be 'sum', got 'r  ©rŠ   )r=   r  r  r  r  r  Úaccumrm  rÆ   rù  rÆ  rÈ  r  r  r  Úaccum_scaleÚaccum_zero_pointÚbinary_op_namer‡  Úunary_op_nameÚunary_op_argsÚunary_op_algorithms                         r8   Úmeta_qconv2d_pointwise_binaryr!    s-   € ð4 ˜UÒ"Ü Ø5°nÐ5EÀQÐGóð ð ˆr:   c                 ó4  — t        | j                  «      }|j                  d   |d<   |	t        j                  t        j                  t        j
                  t        j                  t        j                  fvrt        d|	› �«      ‚| j                  ||	¬«      }|S )Nr3   rœ   zOoutput_dtype must be one of float32, bfloat16, int8, uint8, float8_e4m3fn, got r‘   )
rÄ   r™   rV   r’  r”  rt  r  r•  rŠ   r˜   )r=   r  r  r  r  r  rm  r  r  r  Úpost_op_nameÚpost_op_argsÚpost_op_algorithmr™  rÍ   s                  r8   Úmeta_qlinear_pointwiser&  #  s‰   € ô" ˜AŸG™G“}ˆàŸ7™7 1™:ˆ�RÑØÜ�M‰MÜ�N‰NÜ�J‰JÜ�K‰KÜ×Ñð 
ñ 
ô !ØaÐbnÐaoÐpóð ð �k‰k˜,¨lˆkÓ;ˆØˆ
r:   c                 óB  — |dk(  r|S t        | j                  «      }|j                  d   |d<   |
t        j                  t        j                  t        j
                  t        j                  t        j                  fvrt        d|
› �«      ‚| j                  ||
¬«      }|S )Nr  r3   rœ   r  r‘   )
rÄ   r™   rV   r’  r”  r  rt  r•  rŠ   r˜   )r=   r  r  r  r  r  Úx_2rm  r  r  r  Úx2_scaleÚx2_zpr  r‡  r  r  r   r™  rÍ   s                       r8   Úmeta_qlinear_pointwise_binaryr+  D  s–   € ð, ˜UÒ"ØˆJÜ˜AŸG™G“}ˆàŸ7™7 1™:ˆ�RÑØÜ�M‰MÜ�N‰NÜ�K‰KÜ�J‰JÜ×Ñð 
ñ 
ô !ØaÐbnÐaoÐpóð ð �k‰k˜,¨lˆkÓ;ˆØˆ
r:   c                 óv   — t        | j                  «      }|j                  d   |d<   | j                  |«      }|S )Nr3   rœ   )rÄ   r™   r˜   )r=   r  rm  r™  rÍ   s        r8   Úmeta_linear_dynamic_fp16r-  l  s6   € ô ˜AŸG™G“}ˆàŸ7™7 1™:ˆ�RÑØ�k‰k˜,Ó'ˆØˆ
r:   Ú	quantizedrì  ©r3   c                 ó.  — t        | |||||«      \  }}}| j                  «       dk(  r| j                  d«      nd}	t        j                  }
| j                  «       dk(  r|||g}n|	|||g}t        j
                  || j                  | j                  |
¬«      S ©Nr5  éüÿÿÿr3   r0   r¯  )Ú#max_pool2d_checks_and_compute_shaper†   r£   rV   rà  rŒ   r^   rƒ   ©rØ   rØ  rÆ   rù  rÆ  Ú	ceil_modeÚnInputPlaneÚoutputHeightÚoutputWidthr  r¬   r£   s               r8   Úmeta_quantized_max_pool2dr9  }  s�   € ô 0Ø�; ¨°¸9ó
ñ		
ØØØð $)§9¡9£;°!Ò#3�—‘˜B”¸ˆÜ×+Ñ+ˆØ�9‰9‹;˜!ÒØ ¨{Ð;‰Dà˜K¨°{ÐCˆDÜ�{‰{ØØ—+‘+Ø—<‘<Ø'ô	
ð 	
r:   c                 óþ  ‡ ‡‡‡— t        j                  ‰ j                  «       dk(  ˆ fd„«       t        j                  ‰j                  «       dk(  ˆfd„«       t        j                  ‰ j                  t         j                  t         j
                  t         j                  fv ˆ fd„«       t        j                  ‰j                  t         j                  k(  ˆfd„«       t        j                  ‰j                  t         j                  k(  ˆfd„«       t        j                  ‰j                  ‰ j                  k(  ˆfd„«       ‰ j                  ‰ j                  d«      ‰j                  d«      ‰ j                  ¬	«      S )
Nrý   c                  ó,   •— d‰ j                  «       › d�S )Nzx must be a 2D tensor, got r  r�   ©r=   s   €r8   re   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>œ  ó   ø€ Ð-HÈÏÉËÈ	ÐQRÐ+S€ r:   c                  ó,   •— d‰ j                  «       › d�S )Nzw must be a 2D tensor, got r  r�   ©r  s   €r8   re   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>�  r=  r:   c                  ó"   •— d‰ j                   › �S ©Nz#expected x to be f32/f16/bf16, got r‘   r<  s   €r8   re   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>   s   ø€ Ð9¸!¿'¹'¸ÐC€ r:   c                  ó"   •— d‰ j                   › �S ©Nzexpected w to be uint8, got r‘   r?  s   €r8   re   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>£  s   ø€ Ð.JÈ1Ï7É7È)Ð,T€ r:   c                  ó"   •— d‰ j                   › �S )Nz q_group_size must be int64, got r‘   )Úq_group_sizes   €r8   re   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>§  s   ø€ Ð6°|×7IÑ7IÐ6JÐK€ r:   c                  ó"   •— d‰ j                   › �S )Nz5q_scale_and_zeros must have the same dtype as x, got r‘   )Úq_scale_and_zeross   €r8   re   z/meta_int4mm_packed_weight_cpu.<locals>.<lambda>«  s   ø€ ÐKÐL]×LcÑLcÐKdÐe€ r:   r   r‘   )rV   rg   r†   r^   r’  r“  r”  r  r·   r˜   r£   ©r=   r  rE  rG  s   ````r8   Úmeta_int4mm_packed_weight_cpurI  š  sõ   û€ ä�‰�Q—U‘U“W ‘\Ó#SÔTÜ�‰�Q—U‘U“W ‘\Ó#SÔTÜ�‰Ø�G‰GœŸ™¤u§}¡}´e·n±nÐEÐEÛCô	
ô 	�‰Ø�G‰G”u—{‘{Ñ"Ó$Tô	
ô 	�‰Ø×Ñ¤%§+¡+Ñ-ÛKô	
ô 	�‰Ø×#Ñ# q§w¡wÑ.Ûeô	
ð �{‰{˜1Ÿ6™6 !›9 a§f¡f¨Q£i°q·w±wˆ{Ó?Ð?r:   c                 óŽ   ‡ ‡‡‡— t        j                  ‰ j                  «       ‰k(  xr ‰ j                  ‰   ‰k(  ˆˆˆˆ fd„«       y )Nc                  ój   •— d‰ › d‰› d‰› d�d‰j                  «       › d‰› d‰j                  ‰   › �z   S )NzExpected a tensor of dimension z and tensor.size[z] == rx   zbut got : dimension z] = ©r†   r™   )r†   Údim_sizer£   rb  s   €€€€r8   re   z check_dim_size.<locals>.<lambda>´  sP   ø€ Ð1°#°Ð6GÈÀzÐQVÐW[ÐV\Ð\^Ð_Ø  §¡£ Ð.?À¸zÈÈfÏlÉlÐ[cÑNdÐMeÐ
fñg€ r:   )rV   rg   r†   r™   )rb  r†   rM  r£   s   ````r8   Úcheck_dim_sizerN  ±  s6   û€ Ü	‡L�LØ�
‰
‹˜ÑÒ> §¡¨XÑ 6¸$Ñ >ö	gõr:   r©  rª  r^   c                 ó,   — t        j                  | «      S rD   rJ  )rØ   r©  rª  r^   s       r8   Úmeta_quantize_per_tensorrP  ¹  s   € ô ×Ñ˜EÓ"Ð"r:   c                 óÖ  ‡ — d„ } |d|«      \  }}	t        j                  t        |«      dv d„ «       t        j                  ‰ j                  t         j                  t         j
                  t         j                  t         j                  fvˆ fd„«       t        |«      dk(  r||	}}
n%t        |«      dk(  r|d   |d   }}
n |d|«      \  }
} |d	|«      \  }}t        j                  |d u xs |dk7  d
„ «       ‰ j                  «       dk(  r‰ j                  d«      nd}‰ j                  d«      }‰ j                  d«      }‰ j                  d«      }t        ||||
d|«      }t        ||	||d|«      }t        j                  ‰ «      }t        ‰ ||	|
|||dd||||||«       ‰ j                  «       dk(  r|||g}n||||g}t        j                  |‰ j                  ‰ j                  |¬«      S )Nc                 óˆ   ‡ — t        j                  t        |«      dv ˆ fd„«       |d   }t        |«      dk(  r|n|d   }||fS )N©r3   rý   c                  ó   •— d‰ › d�S )Nzavg_pool2d: ú4 must either be a single int, or a tuple of two intsr4   ©r÷  s   €r8   re   z1meta_avg_pool2d.<locals>.unpack.<locals>.<lambda>Í  ó   ø€ �l 4 &Ð(\Ð]€ r:   r   r3   ©rV   rg   rÃ   ©r÷  rÑ  ÚHÚWs   `   r8   Úunpackzmeta_avg_pool2d.<locals>.unpackÊ  óG   ø€ Ü�‰Ü�‹H˜ÐÛ]ô	
ð �‰FˆÜ�S“˜Q’‰A C¨¡FˆØ�!ˆtˆr:   rØ  ©r   r3   rý   c                   ó   — y©NzOavg_pool2d: stride must either be omitted, a single int, or a tuple of two intsr4   r4   r:   r8   re   z!meta_avg_pool2d.<locals>.<lambda>Ö  rm   r:   c                  ó@   •— d‰ j                   j                  «       › d�S )Nz""avg_pool2d" not implemented for 'r  r  r  s   €r8   re   z!meta_avg_pool2d.<locals>.<lambda>Ú  ó   ø€ Ð6°u·{±{×7JÑ7JÓ7LÐ6MÈQÐQ€ r:   r   r3   rÆ   rù  c                   ó   — y©Nzdivisor must be not zeror4   r4   r:   r8   re   z!meta_avg_pool2d.<locals>.<lambda>ç  rm   r:   r5  r2  éýÿÿÿrò  rœ   r0   r¯  )rV   rg   rÃ   r^   r  Úuint16Úuint32r)  r†   r£   Úpooling_output_shaperO   r   Úpool2d_shape_checkrŒ   rƒ   )rØ   rØ  rÆ   rù  r5  Úcount_include_padÚdivisor_overrider\  ÚkHÚkWÚdHÚdWÚpadHÚpadWr  r6  ÚinputHeightÚ
inputWidthr7  r8  r¬   r£   s   `                     r8   Úmeta_avg_pool2drt  À  sâ  ø€ òñ �M ;Ó/�F€BˆÜ	‡L�LÜˆF‹�yÐ Ùaôô 
‡L�LØ�‰œEŸK™K¬¯©´u·|±|ÄUÇ\Á\ÐRÐRÛQôô ˆ6ƒ{�aÒØ�RˆB‰Ü	ˆV‹˜Ò	Ø˜‘˜F 1™IˆB‰á˜ &Ó)‰ˆˆBá˜	 7Ó+�J€Dˆ$ä	‡L�LØ˜DÐ Ò9Ð$4¸Ñ$9Ù*ôð
  %Ÿy™y›{¨aÒ/ˆU�Z‰Z˜Œ^°Q€FØ—*‘*˜R“.€KØ—*‘*˜R“.€KØ—‘˜B“€Jä'¨°R¸¸rÀ1ÀiÓP€LÜ& z°2°t¸RÀÀIÓN€Kä×/Ñ/°Ó6€MÜØØ
Ø
Ø
Ø
ØØØ	Ø	ØØØØØØôð$ ‡y�yƒ{�aÒØ˜\¨;Ð7‰à˜ \°;Ð?ˆÜ�;‰;ØØ�k‰kØ�|‰|Ø#ô	ð r:   c                 óÀ   — t        | ||||||dd|	|
||||«       | j                  «       }|	}t        |||dz
  |«       t        |||dz
  |«       t        |||dz
  |«       y )Nr3   r0   rý   )ri  r†   rN  )rØ   Ú
gradOutputr  rl  rm  rn  ro  rp  rq  r6  rr  rs  r7  r8  Ú
mem_formatr¥   ÚnOutputPlanes                    r8   Úavg_pool2d_backward_shape_checkry    s�   € ô" ØØ
Ø
Ø
Ø
ØØØ	Ø	ØØØØØØôð$ �9‰9‹;€DØ€Lä�:˜t T¨A¡X¨|Ô<Ü�:˜t T¨A¡X¨|Ô<Ü�:˜t T¨A¡X¨{Õ;r:   c                 óÐ  — t        j                  t        |«      dk(  xs t        |«      dk(  d„ «       |d   }t        |«      dk(  r|n|d   }	t        j                  t        |«      dk(  xs t        |«      dk(  xs t        |«      dk(  d„ «       t        |«      dk(  r|n|d   }
t        |«      dk(  r|	nt        |«      dk(  r|
n|d   }t        j                  t        |«      dk(  xs t        |«      dk(  d„ «       |d   }t        |«      dk(  r|n|d   }t        j                  |d u xs |dk7  d„ «       |j                  }|j	                  «       dk(  r|d	   nd}|d
   }|d   }|d   }t        ||||
d|«      }t        ||	||d|«      }t        j                  |«      }t        || |||	|
|||||||||«       t        j                  ||j                  |j                  |¬«      S )Nr3   rý   c                   ó   — y)NzKavg_pool2d: kernel_size must either be a single int, or a tuple of two intsr4   r4   r:   r8   re   z*meta_avg_pool2d_backward.<locals>.<lambda>L  rm   r:   r   c                   ó   — yr`  r4   r4   r:   r8   re   z*meta_avg_pool2d_backward.<locals>.<lambda>R  rm   r:   c                   ó   — y)NzGavg_pool2d: padding must either be a single int, or a tuple of two intsr4   r4   r:   r8   re   z*meta_avg_pool2d_backward.<locals>.<lambda>X  rm   r:   c                   ó   — yrd  r4   r4   r:   r8   re   z*meta_avg_pool2d_backward.<locals>.<lambda>_  rm   r:   r5  r2  re  rò  rœ   r¯  )rV   rg   rÃ   r™   r†   rh  rO   r   ry  rŒ   r^   rƒ   )ÚgradOutput_rØ   rØ  rÆ   rù  r5  rj  rk  rl  rm  rn  ro  rp  rq  Ú
input_sizer  r6  rr  rs  r7  r8  rw  s                         r8   Úmeta_avg_pool2d_backwardr�  >  s  € ô 
‡L�LÜˆKÓ˜AÑÒ6¤ [Ó!1°QÑ!6Ù]ôð 
�Q‰€BÜ�;Ó 1Ò$‰¨+°a©.€BÜ	‡L�LÜˆF‹�qÑÒ@œC ›K¨1Ñ,Ò@´°F³¸qÑ0@Ùaôô �6‹{˜aÒ‰ V¨A¡Y€BÜ�6‹{˜aÒ‰¬3¨v«;¸!Ò+;¡RÀÈÁ€BÜ	‡L�LÜˆG‹˜ÑÒ.œS ›\¨QÑ.ÙYôð �1‰:€DÜ�w“< 1Ò$‰4¨'°!©*€Dä	‡L�LØ˜DÐ Ò9Ð$4¸Ñ$9Ù*ôð
 —‘€JØ$Ÿy™y›{¨aÒ/ˆZ˜Š^°Q€FØ˜R‘.€KØ˜R‘.€KØ˜B‘€Jä'¨°R¸¸rÀ1ÀiÓP€LÜ& z°2°t¸RÀÀIÓN€Kä×,Ñ,¨UÓ3€Jä#ØØØØ
Ø
Ø
Ø
ØØØØØØØØôô$ �;‰;ØØ�k‰kØ�|‰|Ø ô	ð r:   c                 ó  ‡ — t        j                  t        |«      dv d„ «       |d   }t        |«      dk(  r|n|d   }t        |«      dk(  r|n|d   }	t        j                  | xs t        |«      dv d„ «       t        j                  ‰ j                  t         j                  t         j
                  t         j                  t         j                  fvˆ fd„«       |s|n|d   }
|s|nt        |«      dk(  r|
n|d   }|s|	nt        |«      dk(  r|
n|d   }t        j                  t        |«      dv d„ «       |d   }t        |«      dk(  r|n|d   }t        |«      dk(  r|n|d   }t        j                  ‰ j                  d	v d
„ «       t        j                  | xs |dk7  d„ «       ‰ j                  d«      }‰ j                  d«      }‰ j                  d«      }‰ j                  d«      }‰ j                  d«      }t        ||||
d|«      }t        ||||d|«      }t        ||	||d|«      }t        ‰ ||||	|
|||||ddd||||||dd¬«       ‰ j                  dk(  r‰ j                  ||||f«      S ‰ j                  |||||f«      S )N©r3   r0   c                   ó   — y©NzFavg_pool3d: kernel_size must be a single int, or a tuple of three intsr4   r4   r:   r8   re   z!meta_avg_pool3d.<locals>.<lambda>”  rm   r:   r   r3   rý   c                   ó   — y©NzJavg_pool3d: stride must be omitted, a single int, or a tuple of three intsr4   r4   r:   r8   re   z!meta_avg_pool3d.<locals>.<lambda>œ  rm   r:   c                  ó@   •— d‰ j                   j                  «       › d�S )Nz""avg_pool3d" not implemented for 'r  r  r  s   €r8   re   z!meta_avg_pool3d.<locals>.<lambda>   rb  r:   c                   ó   — y©NzBavg_pool3d: padding must be a single int, or a tuple of three intsr4   r4   r:   r8   re   z!meta_avg_pool3d.<locals>.<lambda>¨  rm   r:   ©r5  rZ  c                   ó   — y©Nz9non-empty 4D or 5D (batch mode) tensor expected for inputr4   r4   r:   r8   re   z!meta_avg_pool3d.<locals>.<lambda>°  rm   r:   c                   ó   — yrd  r4   r4   r:   r8   re   z!meta_avg_pool3d.<locals>.<lambda>µ  rm   r:   r2  re  rò  rœ   zavg_pool3d()T)Úcheck_input_sizer5  )rV   rg   rÃ   r^   r  rf  rg  r)  r¥   r£   rh  Úpool3d_shape_checkr˜   )rØ   rØ  rÆ   rù  r5  rj  rk  ÚkTrl  rm  ÚdTrn  ro  ÚpadTrp  rq  r  ÚnslicesÚitimeÚiheightÚiwidthÚotimeÚoheightÚowidths   `                       r8   Úmeta_avg_pool3dr›  ‡  sŒ  ø€ ô 
‡L�LÜˆKÓ˜FÐ"ÙXôð 
�Q‰€BÜ�;Ó 1Ò$‰¨+°a©.€BÜ�;Ó 1Ò$‰¨+°a©.€Bä	‡L�LØˆ
Ò+”c˜&“k VÐ+Ù\ôô 
‡L�LØ�‰œEŸK™K¬¯©´u·|±|ÄUÇ\Á\ÐRÐRÛQôñ ‰˜v a™y€BÙ‰¤c¨&£k°QÒ&6¡¸FÀ1¹I€BÙ‰¤c¨&£k°QÒ&6¡¸FÀ1¹I€Bä	‡L�LÜˆG‹˜ÐÙTôð �1‰:€DÜ�w“< 1Ò$‰4¨'°!©*€DÜ�w“< 1Ò$‰4¨'°!©*€Dä	‡L�LØ�
‰
�fÐÙKôô
 
‡L�LØÐÒ5Ð 0°AÑ 5Ù*ôð
 �Z‰Z˜‹]€FØ�j‰j˜‹n€GØ�J‰J�r‹N€EØ�j‰j˜‹n€GØ�Z‰Z˜‹^€Fä  ¨¨D°"°a¸ÓC€EÜ" 7¨B°°b¸!¸YÓG€GÜ! &¨"¨d°B¸¸9ÓE€FäØØØ
Ø
Ø
Ø
Ø
Ø
ØØØØ	Ø	Ø	ØØØØØØØØõ-ð2 ‡z�z�Q‚Ø�‰ ¨°¸Ð@ÓAÐAà�‰ ¨°¸ÀÐHÓIÐIr:   c                 óæ  — t        j                  t        |«      dv d„ «       |d   }t        |«      dk(  r|n|d   }	t        |«      dk(  r|n|d   }
t        j                  | xs t        |«      dv d„ «       |s|n|d   }|s|	nt        |«      dk(  r|n|d   }|s|
nt        |«      dk(  r|n|d   }t        j                  t        |«      dv d„ «       |d   }t        |«      dk(  r|n|d   }t        |«      dk(  r|n|d   }t        j                  |j                  dv d	„ «       t        j                  | xs |dk7  d
„ «       |j	                  d«      }|j	                  d«      }|j	                  d«      }|j	                  d«      }t        ||||d|«      }t        ||	||d|«      }t        ||
||d|«      }t        || |||	|
||||||||||||d«       |j                  |j                  «      S )Nrƒ  c                   ó   — yr…  r4   r4   r:   r8   re   z*meta_avg_pool3d_backward.<locals>.<lambda>ï  rm   r:   r   r3   rý   c                   ó   — yr‡  r4   r4   r:   r8   re   z*meta_avg_pool3d_backward.<locals>.<lambda>÷  rm   r:   c                   ó   — yrŠ  r4   r4   r:   r8   re   z*meta_avg_pool3d_backward.<locals>.<lambda>ÿ  rm   r:   r‹  c                   ó   — yr�  r4   r4   r:   r8   re   z*meta_avg_pool3d_backward.<locals>.<lambda>  rm   r:   c                   ó   — yrd  r4   r4   r:   r8   re   z*meta_avg_pool3d_backward.<locals>.<lambda>  rm   r:   r2  re  rò  rœ   zavg_pool3d_backward())	rV   rg   rÃ   r¥   r£   rh  Úavg_pool3d_backward_shape_checkr˜   r™   )r*  rØ   rØ  rÆ   rù  r5  rj  rk  r‘  rl  rm  r’  rn  ro  r“  rp  rq  r”  r•  r–  r—  Úotime_for_shape_checkÚoheight_for_shape_checkÚowidth_for_shape_checks                           r8   Úmeta_avg_pool3d_backwardr¦  á  s  € ô 
‡L�LÜˆKÓ˜FÐ"ÙXôð 
�Q‰€BÜ�;Ó 1Ò$‰¨+°a©.€BÜ�;Ó 1Ò$‰¨+°a©.€Bä	‡L�LØˆ
Ò+”c˜&“k VÐ+Ù\ôñ ‰˜v a™y€BÙ‰¤c¨&£k°QÒ&6¡¸FÀ1¹I€BÙ‰¤c¨&£k°QÒ&6¡¸FÀ1¹I€Bä	‡L�LÜˆG‹˜ÐÙTôð �1‰:€DÜ�w“< 1Ò$‰4¨'°!©*€DÜ�w“< 1Ò$‰4¨'°!©*€Dä	‡L�LØ�
‰
�fÐÙKôô
 
‡L�LØÐÒ5Ð 0°AÑ 5Ù*ôð
 �j‰j˜‹n€GØ�J‰J�r‹N€EØ�j‰j˜‹n€GØ�Z‰Z˜‹^€Fä0°¸¸DÀ"ÀaÈÓSÐÜ2°7¸BÀÀbÈ!ÈYÓWÐÜ1°&¸"¸dÀBÈÈ9ÓUÐä#ØØØØ
Ø
Ø
Ø
Ø
Ø
ØØØØØØØØØØô'ð, �?‰?˜5Ÿ;™;Ó'Ð'r:   c                 ó,  ‡ — t        j                  ‰ j                  dk(  xs ‰ j                  dk(  ˆ fd„«       ‰ j                  d d t	        |«      z   }t        j                  ‰ «      }t        j                  |‰ j                  ‰ j                  |¬«      S )Nr0   r5  c                  ó"   •— d‰ j                   › �S )Nz"Expected 3D or 4D tensor, but got r$  r±   s   €r8   re   z*meta_adaptive_avg_pool2d.<locals>.<lambda>5  ó   ø€ Ð4°T·Z±Z°LÐA€ r:   rò  r¯  )
rV   rg   r¥   r™   rf   rO   r   rŒ   r^   rƒ   )rš   Úoutput_sizer™  r¬   s   `   r8   Úmeta_adaptive_avg_pool2dr«  1  s|   ø€ ä	‡L�LØ�	‰	�Q‰Ò(˜$Ÿ)™) q™.ÛAôð —:‘:˜c˜r�?¤U¨;Ó%7Ñ7€LÜ×/Ñ/°Ó5€Mô �;‰;ØØ�j‰jØ�{‰{Ø#ô	ð r:   c                 óÆ   ‡ — t        j                  ‰ j                  dk(  xs ‰ j                  dk(  ˆ fd„«       ‰ j                  ‰ j                  d d t        |«      z   «      S )Nr5  rZ  c                  ó"   •— d‰ j                   › �S )Nz"Expected 4D or 5D tensor, but got r$  r±   s   €r8   re   z*meta_adaptive_avg_pool3d.<locals>.<lambda>G  r©  r:   re  )rV   rg   r¥   r˜   r™   rf   )rš   rª  s   ` r8   Úmeta_adaptive_avg_pool3dr®  C  sO   ø€ ä	‡L�LØ�	‰	�Q‰Ò(˜$Ÿ)™) q™.ÛAôð �>‰>˜$Ÿ*™* S b˜/¬E°+Ó,>Ñ>Ó?Ð?r:   c                 óî  ‡ ‡‡— ‰ j                   }t        d|«      D ].  Št        j                  ‰ j	                  ‰«      dkD  ˆ ˆfd„«       Œ0 t        j                  |dk(  xs |dk(  ˆfd„«       t        j                  ‰j
                  ‰ j
                  k(  ˆ ˆfd„«       t        j                  }t        ‰«      rt        j                  }‰j                  ‰j                  «      j                  |¬«      S )	Nr3   r   c                  ó*   •— d‰ j                   › d‰› d�S )Nz{adaptive_avg_pool2d_backward(): Expected grad_output to have non-zero                       size for non-batch dimensions, ú with dimension ú being emptyr$  )Úgrad_outrÜ   s   €€r8   re   z4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>R  s&   ø€ ð 6Ø6>·n±nÐ5EÐEUÐVWÐUXÐXdðf€ r:   r0   r5  c                  ó"   •— d‰ j                   › �S )NzBadaptive_avg_pool2d_backward(): Expected 3D or 4D tensor, but got r$  r±   s   €r8   re   z4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>W  s   ø€ ÐTÐUY×U_ÑU_ÐT`Ða€ r:   c                  ó<   •— d‰j                   › d‰ j                   › �S ©Núexpected dtype z! for `grad_output` but got dtype r‘   )r³  rš   s   €€r8   re   z4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>[  s   ø€ �/ $§*¡* Ð-NÈxÏ~É~ÐN^Ð_€ r:   r«   )r¥   rÅ   rV   rg   r£   r^   r°   rã  rà  r˜   r™   rY  )r³  rš   r¥   r¬   rÜ   s   ``  @r8   Ú"meta__adaptive_avg_pool2d_backwardr¸  L  sÇ   ú€ à�=‰=€DÜ�1�d‹^ò 
ˆÜ�‰Ø�M‰M˜!Ó˜qÑ ôfõ	
ð
ô 
‡L�LØ�‰	Ò�T˜Q‘YÛaôô 
‡L�LØ�
‰
�h—n‘nÑ$Ü_ôô ×+Ñ+€MÜ˜ÔÜ×+Ñ+ˆØ�>‰>˜$Ÿ*™*Ó%×(Ñ(°}Ð(ÓEÐEr:   c                 ód   — t        | d«       t        j                  |t        j                  ¬«      S )NÚadaptive_avg_pool3d_backwardr«   )Ú!_adaptive_pool_empty_output_checkrV   r¯   r?  ©r*  rš   s     r8   Ú"meta__adaptive_avg_pool3d_backwardr½  c  s(   € ô & kÐ3QÔRÜ×Ñ˜D´×0NÑ0NÔOÐOr:   r*  c                 óž   ‡ ‡‡— ‰ j                   }t        d|«      D ]/  Št        j                  ‰ j	                  ‰«      dkD  ˆˆ ˆfd„«       Œ1 y )Nr3   r   c                  ó.   •— ‰ › d‰j                   › d‰› d�S )Nzc(): Expected grad_output to have non-zero size for non-batch dimensions, but grad_output has sizes r±  r²  r$  )r  r*  rÜ   s   €€€r8   re   z3_adaptive_pool_empty_output_check.<locals>.<lambda>o  s/   ø€ Ø�*ð -Ø-8×->Ñ->Ð,?Ð?OÐPQÈsÐR^ð`ð r:   )r¥   rÅ   rV   rg   r£   )r*  r  r¥   rÜ   s   `` @r8   r»  r»  j  sG   ú€ Ø×Ñ€DÜ�1�d‹^ò 
ˆÜ�‰Ø×Ñ˜QÓ !Ñ#õõ	
ñ
r:   c                 ó  ‡ ‡— ‰ j                   }t        j                  |dv ˆ fd„«       t        d|«      D ].  Št        j                  ‰ j	                  ‰«      dkD  ˆˆ fd„«       Œ0 t        j                  t        |«      dk(  d„ «       d}d}d}‰ j                   dk(  r‰ j	                  d«      }|dz  }‰ j	                  |dz
  «      }|\  }}‰ j                   d	k(  r;|||f}‰ j                  |«      }	‰ j                  |t        j                  ¬
«      }
|	|
fS ||||f}t        j                  ‰ «      }‰ j                  |«      j                  |¬«      }	‰ j                  |t        j                  ¬
«      j                  |¬«      }
|	|
fS )N©r0   r5  c                  ó"   •— d‰ j                   › �S )Nz:adaptive_max_pool2d(): Expected 3D or 4D tensor, but got: r$  r  s   €r8   re   z*meta_adaptive_max_pool2d.<locals>.<lambda>|  ó   ø€ ÐLÈUÏ[É[ÈMÐZ€ r:   r3   r   c                  ó*   •— d‰j                   › d‰ › d�S )Nzjadaptive_max_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes r±  r²  r$  ©rÜ   rØ   s   €€r8   re   z*meta_adaptive_max_pool2d.<locals>.<lambda>�  ó%   ø€ ð'Ø',§{¡{ mÐ3CÀAÀ3ÀlðTð r:   rý   c                   ó   — y)NzCadaptive_max_pool2d(): internal error: output_size.size() must be 2r4   r4   r:   r8   re   z*meta_adaptive_max_pool2d.<locals>.<lambda>‰  rm   r:   r5  r0   r‘   r«   )r¥   rV   rg   rÅ   r£   rÃ   r˜   r·   rO   r   rY  )rØ   rª  r¥   ÚdimHÚsizeBÚsizeDÚosizeHÚosizeWr¨   rÍ   r´   r¬   rÜ   s   `           @r8   Úmeta_adaptive_max_pool2drÍ  v  s|  ù€ ð �:‰:€DÜ	‡L�LØ�ˆÛZôô �1�d‹^ò 
ˆÜ�‰Ø�J‰J�q‹M˜AÑôõ	
ð
ô 
‡L�LÜˆKÓ˜AÑÙUôð
 €DØ€EØ€Eà‡z�z�Q‚Ø—
‘
˜1“ˆØ�‰	ˆà�J‰J�t˜a‘xÓ €EØ �N€FˆFà‡z�z�Q‚Ø˜F FÐ+ˆ	Ø�o‰o˜iÓ(ˆØ—/‘/ )´5·;±;�/Ó?ˆØ�Gˆ|Ðà˜E 6¨6Ð2ˆ	Ü×3Ñ3°EÓ:ˆØ�o‰o˜iÓ(×+Ñ+¸-Ð+ÓHˆØ—/‘/ )´5·;±;�/Ó?×BÑBØ'ð Có 
ˆð �Gˆ|Ðr:   c                 óN  ‡ ‡— ‰ j                   }t        j                  |dv ˆ fd„«       t        ‰ d«       t        j                  ‰j                  ‰ j                  k(  ˆ ˆfd„«       t        j                  ‰«      }‰j                  ‰j                  «      j                  |¬«      S )NrÁ  c                  ó"   •— d‰ j                   › �S )NzKadaptive_max_pooling2d_backward(): Expected 3D or 4D grad_output, but got: r$  ©r*  s   €r8   re   z3meta_adaptive_max_pool2d_backward.<locals>.<lambda>¬  s   ø€ Ð]Ð^i×^oÑ^oÐ]pÐq€ r:   Úadaptive_max_pool2d_backwardc                  ó<   •— d‰j                   › d‰ j                   › �S r¶  r‘   )r*  rØ   s   €€r8   re   z3meta_adaptive_max_pool2d_backward.<locals>.<lambda>³  s!   ø€ �/ %§+¡+ Ð.OÐP[×PaÑPaÐObÐc€ r:   r«   )
r¥   rV   rg   r»  r^   rO   r   r˜   r™   rY  )r*  rØ   r´   r¥   r¬   s   ``   r8   Ú!meta_adaptive_max_pool2d_backwardrÓ  ¦  s‡   ù€ ð ×Ñ€DÜ	‡L�LØ�ˆÛqôô
 & kÐ3QÔRä	‡L�LØ�‰�{×(Ñ(Ñ(Ücôô
 ×/Ñ/°Ó6€MØ�?‰?˜5Ÿ;™;Ó'×*Ñ*¸Ð*ÓGÐGr:   c                 ó  ‡ ‡— ‰ j                   }t        j                  |dv ˆ fd„«       t        d|«      D ].  Št        j                  ‰ j	                  ‰«      dkD  ˆˆ fd„«       Œ0 t        j                  t        |«      dk(  d„ «       d}d}d}|dk(  r‰ j	                  d«      }|dz  }‰ j	                  |«      }|\  }}}|d	k(  r||||f}	n|||||f}	‰ j                  |	«      }
‰ j                  |	t        j                  ¬
«      }|
|fS )Nr‹  c                  ó"   •— d‰ j                   › �S )Nz:adaptive_max_pool3d(): Expected 4D or 5D tensor, but got: r$  r  s   €r8   re   z*meta_adaptive_max_pool3d.<locals>.<lambda>À  rÃ  r:   r3   r   c                  ó*   •— d‰j                   › d‰ › d�S )Nzjadaptive_max_pool3d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes r±  r²  r$  rÅ  s   €€r8   re   z*meta_adaptive_max_pool3d.<locals>.<lambda>Å  rÆ  r:   r0   c                   ó   — y)NzCadaptive_max_pool3d(): internal error: output_size.size() must be 3r4   r4   r:   r8   re   z*meta_adaptive_max_pool3d.<locals>.<lambda>Í  rm   r:   rZ  r5  r‘   )r¥   rV   rg   rÅ   r£   rÃ   r˜   r·   )rØ   rª  r¥   ÚdimDrÉ  rÊ  ÚosizeTrË  rÌ  r¨   rÍ   r´   rÜ   s   `           @r8   Úmeta_adaptive_max_pool3drÚ  º  s  ù€ ð �:‰:€DÜ	‡L�LØ�ˆÛZôô �1�d‹^ò 
ˆÜ�‰Ø�J‰J�q‹M˜AÑôõ	
ð
ô 
‡L�LÜˆKÓ˜AÑÙUôð
 €DØ€EØ€Eàˆq‚yØ—
‘
˜1“ˆØ�‰	ˆà�J‰J�tÓ€EØ(Ñ€FˆF�Fàˆq‚yØ˜F F¨FÐ3‰	à˜E 6¨6°6Ð:ˆ	à
�/‰/˜)Ó
$€CØ�o‰o˜i¬u¯{©{ˆoÓ;€Gà�ˆ<Ðr:   c                 óP   — t        | d«       |j                  |j                  «      S )NÚadaptive_max_pool3d_backward)r»  r˜   r™   )r*  rØ   r´   s      r8   Ú!meta_adaptive_max_pool3d_backwardrÝ  æ  s"   € ô & kÐ3QÔRØ�?‰?˜5Ÿ;™;Ó'Ð'r:   c                 ó>   — |€t        d«      ‚| j                  |«      S )Nz:cannot repeat_interleave a meta tensor without output_size)rX  r˜   )Úrepeatsrª  s     r8   Úmeta_repeat_interleave_Tensorrà  í  s%   € àÐÜÐWÓXÐXØ×Ñ˜[Ó)Ð)r:   c                 ó~  — | j                   j                  st        d| j                   › �«      ‚|j                   j                  st        d|j                   › �«      ‚t        | j	                  t        | j                   «      «      |j	                  t        |j                   «      «      t        j                  ¬«      }|S )Nz!real must be floating point, got z!imag must be floating point, got ©rL   )r^   r  rŠ   rT   rY  r   r   rP   )ÚrealÚimagr  s      r8   Úmeta_complexrå  ô  s’   € ð �:‰:×'Ò'ÜÐ@ÀÇÁÀÐMÓNÐNØ�:‰:×'Ò'ÜÐ@ÀÇÁÀÐMÓNÐNÜØ�‰Ô+¨D¯J©JÓ7Ó8Ø�‰Ô+¨D¯J©JÓ7Ó8Ü6×>Ñ>ô€Fð
 €Mr:   )Ú
fill_valueræ  c                ó  — t        | «      dv r1| j                  || j                  «       ft        j                  ¬«      S t        j
                  || j                  «       fd|ft        j                  | j                  ¬«      S )N)rè   Úmpsr   r‘   r3   ©r^   rƒ   )rð   r˜   r†   rV   r•   rX  rƒ   )rš   r£   ræ  s      r8   Únonzero_staticrê    sj   € ô �4ÓÐ1Ñ1Ø�~‰~˜t T§X¡X£ZÐ0¼¿
¹
ˆ~ÓCÐCä×"Ñ"Ø�4—8‘8“:ÐØ�ˆIÜ—*‘*Ø—;‘;ô	
ð 	
r:   c                 ó
  — t        j                  t        j                  d„ «       t        j                  | j                  «       | j                  «       fd| j                  «       ft         j                  | j                  ¬«      S )Nc                   ó   — y)NaY  The register_meta function for torch.nonzero() raises unimplemented by default, as a correct data-independent implementation does not exist. This implementation returns a fake value, assuming all elements of the tensor are non-zero. To enable this registration, please set 'torch.fx.experimental._config.meta_nonzero_assume_all_nonzero' to True.r4   r4   r:   r8   re   znonzero.<locals>.<lambda>  rm   r:   r3   ré  )	rV   Ú_check_not_implementedr�  Úmeta_nonzero_assume_all_nonzerorX  r—   r†   r•   rƒ   r±   s    r8   Únonzerorï    sf   € ô 
× Ñ Ü×2Ñ2ñ	Sôô ×ÑØ	�‰‹�t—x‘x“zÐ"Ø	
ˆD�J‰J‹LÐÜ�j‰jØ�{‰{ô	ð r:   c           
      óê  ‡ ‡‡‡‡‡‡‡‡— t        j                  t        ‰«      d„ «       g }t        ‰«      D �]€  \  ŠŠ‰��ft        j                  ‰j                  t         j
                  t         j                  t         j                  t         j                  fv d„ «       ‰j                  t         j                  t         j                  fv rÈ‰j                  «       }t        |«      Št        j                  ‰‰j                  z   ‰ j                  k  ˆ fd„«       t        ‰j                  «      D ]`  Št        j                  ‰j                  ‰   ‰ j                  ‰‰z      k(  ˆˆˆˆˆ fd„«       |j                  |j                  d‰«      «       Œb �Œ]|j                  ‰«       �Œp|j                  ‰«       �Œƒ |Št        j                  t        ‰«      ‰ j                  k  ˆˆ fd„«       dd lm} t%         |j&                  ‰Ž «      Št        ‰«      ‰ j                  k  r*‰j                  d «       t        ‰«      ‰ j                  k  rŒ*d}d}‰D ]  Š|dk(  r‰€Œd}Œ|dk(  r‰�Œd	}Œ‰€Œ n d
}|s‡g }g }t        ‰«      D ]*  \  ŠŠ‰€Œ	|j                  ‰«       |j                  ‰«       Œ, t        ‰«      D ]*  \  ŠŠ‰�Œ	|j                  ‰«       |j                  ‰«       Œ, ‰ j)                  |«      Š |Šg Šg Šg Št        ‰«      D ]\  \  }	Š‰€@‰r‰j                  ‰ j                  |	   «       Œ)‰j                  ‰ j                  |	   «       ŒHt%        ‰j                  «      ŠŒ^ ˆˆˆfd„}
‰ j+                  ‰‰z   ‰z   «      }ddlm}  |‰ j1                  «       dk(  «      r|S  |
‰ «      }t3        j4                  |«      \  }}t%        |«      t%        t        t        |«      «      «      k7  r~t3        j6                  |j                  |«      }t3        j8                  |«      }t3        j6                  |t3        j:                  |«      «      }|j=                  |j?                  «       |«      }|S )Nc                   ó   — y)Nz#at least one index must be providedr4   r4   r:   r8   re   z#meta_index_Tensor.<locals>.<lambda>'  rm   r:   c                   ó   — y)Nz?tensors used as indices must be long, int, byte or bool tensorsr4   r4   r:   r8   re   z#meta_index_Tensor.<locals>.<lambda>/  rm   r:   c                  ó"   •— d‰ j                   › �S )Nú)too many indices for tensor of dimension r7  r±   s   €r8   re   z#meta_index_Tensor.<locals>.<lambda>6  s   ø€ ÐGÈÏ	É	À{ÐS€ r:   c            	      óN   •— d‰j                   › d‰ › d‰j                   › d‰‰z   › �S )NzThe shape of the mask ú
 at index z0 does not match the shape of the indexed tensor r$  )rÜ   r“   Újrw  rš   s   €€€€€r8   re   z#meta_index_Tensor.<locals>.<lambda>;  s<   ø€ Ð"8¸¿¹¸ÀZÐPQÈsð SJØJNÏ*É*ÈÐU_Ð`aÐdeÑ`eÐ_fð!h€ r:   r3   c                  ó<   •— d‰j                   › dt        ‰ «      › d�S )Nrô  z (got ry   )r¥   rÃ   )r´   rš   s   €€r8   re   z#meta_index_Tensor.<locals>.<lambda>F  s!   ø€ Ð;¸D¿I¹I¸;ÀfÌSÐQXË\ÈNÐZ[Ð\€ r:   r   Frý   Tc                 óÞ   •— ‰‰z   ‰z   }t        | j                  «       «      }dgt        ‰«      z  |t        ‰«      t        | j                  «      t        ‰«      z
   | j	                  ||«      S )zI
        This follows restride_src in TensorAdvancedIndexing.cpp
        r   )rÄ   rÆ   rÃ   r™   Ú
as_strided)rš   r™   rí   Úafter_shapeÚbefore_shapeÚreplacement_shapes      €€€r8   Ú_restride_srcz(meta_index_Tensor.<locals>._restride_src‡  so   ø€ ð Ð0Ñ0°;Ñ>ˆÜ�t—{‘{“}Ó%ˆàKLÈ#ÔPSØóQ
ñ K
ˆ”�LÓ!¤C¨¯
©
£O´c¸+Ó6FÑ$FÐGð �‰˜u gÓ.Ð.r:   ró  ) rV   rg   r   Ú	enumerater^   r•   rã   rt  rï  rÃ   r–   r¥   rÅ   r™   rÔ  ÚselectÚtorch._refsrZ  rÄ   r&   rÈ   r˜   rV  rô  r—   rO   Ú3compute_elementwise_output_logical_to_physical_permÚ
apply_permr   Úinvert_permrú  r£   )rš   r´   r  rï  ÚrefsÚstateÚhas_contiguous_subspacer¾  Útransposed_indicesr†   rþ  rÍ   rô  Úrestrided_selfÚpermrR   Ú
perm_shapere  rû  rü  rÜ   r“   r÷  rw  rý  s   ``                @@@@@@@r8   Úmeta_index_Tensorr  %  sÙ  ÿø€ ä	‡L�L”�g“Ñ MÔNð #%€FÜ˜gÓ&ó !‰ˆˆ5ØÑÜ�L‰LØ—‘¤§
¡
¬E¯I©I´u·z±zÄ5Ç:Á:ÐNÐNÙYôð �{‰{œuŸz™z¬5¯:©:Ð6Ñ6ØŸ-™-›/�Ü˜“K�Ü×"Ñ"Ø˜Ÿ
™
‘N d§i¡iÑ/ÛSôô ˜uŸz™zÓ*ò 8�AÜ×&Ñ&ØŸ™ A™¨$¯*©*°Q¸±UÑ*;Ñ;÷hôð
 —M‘M '§.¡.°°AÓ"6Õ7ò8ð —‘˜eÖ$à�M‰M˜%Ö ð/!ð0 €GÜ	‡L�LÜˆG‹˜Ÿ	™	Ñ!Ü\ôõ
 äÐ(�4×(Ñ(¨'Ð2Ó3€Gä
ˆg‹,˜Ÿ™Ò
"Ø�‰�tÔô ˆg‹,˜Ÿ™Ó
"ð €EØ#ÐØò 'ˆØ�AŠ:ØÑ Ø‘Ø�aŠZØ‰}Ø‘àÑ Ùð'ð #'Ðñ
 #ØˆØÐÜ! 'Ó*ò 	1‰HˆAˆuØÑ Ø—‘˜A”Ø"×)Ñ)¨%Õ0ð	1ô " 'Ó*ò 	1‰HˆAˆuØ‰}Ø—‘˜A”Ø"×)Ñ)¨%Õ0ð	1ð �|‰|˜DÓ!ˆØ$ˆð !€LØ€KØ#%ÐÜ Ó(ò 2‰
ˆˆUØˆ=Ù Ø×"Ñ" 4§:¡:¨c¡?Õ3à×#Ñ# D§J¡J¨s¡OÕ4ä $ U§[¡[Ó 1Ñð2ö
/ð �.‰.˜Ð(9Ñ9¸KÑGÓ
H€CåDá�d—j‘j“l aÑ'Ô(àˆ
ñ
 # 4Ó(€NÜ×GÑGÈÓW�G€Dˆ!ô ˆDƒz”Tœ%¤ D£	Ó*Ó+Ò+Ü×%Ñ% c§i¡i°Ó6ˆ
Ü×6Ñ6°zÓBˆ
Ü×%Ñ% j´%×2CÑ2CÀDÓ2IÓJˆ
Ø�n‰n˜SŸX™X›Z¨Ó4ˆØ€Jr:   c                 ó,  — d }d }d }d„ } |||«      }|
d   r/| j                  |j                  «       «      j                  |¬«      }|
d   r/| j                  |j                  «       «      j                  |¬«      }|
d   r| j                  |«      }|||fS )Nc                 ó&  — t        | «      }t        |«      }|t        j                  k(  s|t        j                  k(  rt        j                  S |t        j                  k(  s|t        j                  k(  rt        j                  S t        j                  S rD   )r   rV   rà  rû  r°   )Út1Út2Úfmt1Úfmt2s       r8   Ú_conv_memory_formatz6meta_convolution_backward.<locals>._conv_memory_formatÄ  sp   € Ü$ RÓ(ˆÜ$ RÓ(ˆØ”5×&Ñ&Ò&¨$´%×2EÑ2EÒ*EÜ×&Ñ&Ð&Ø”5×)Ñ)Ò)¨T´U×5KÑ5KÒ-KÜ×)Ñ)Ð)Ü×&Ñ&Ð&r:   r   r«   r3   rý   )r˜   r£   rY  )Úgrad_output_Úinput_Úweight_Úbias_sizes_optrÆ   rù  rÆ  Ú
transposedrÉ  rÈ  Úoutput_maskÚbackend_grad_inputÚbackend_grad_weightÚbackend_grad_biasr  r¬   s                   r8   Úmeta_convolution_backwardr  ª  s¸   € ð  ÐØÐØÐò'ñ (¨°Ó8€MØ�1‚~Ø)×3Ñ3°F·K±K³MÓB×EÑEØ'ð Fó 
Ðð �1‚~Ø*×4Ñ4°W·\±\³^ÓD×GÑGØ'ð Hó 
Ðð �1‚~Ø(×2Ñ2°>ÓBÐàÐ 3Ð5FÐGÐGr:   c                ó¸  ‡‡— ‰j                  d«      }‰j                  d«      }| j                  ||f«      } t        j                  ‰j	                  «       dk(  d„ «       t        j                  ‰j	                  «       dk(  d„ «       t        j                  ‰j                  d«      ‰j                  d«      k(  ˆˆfd„«       t        j                  ‰j                  d«      ‰j                  d«      k(  ˆˆfd„«       t        j                  | j                  d«      |k(  xr | j                  d«      |k(  d	„ «       | j                  | j                  «       «      S )
Nr3   rý   r0   c                   ó   — yr‚  r4   r4   r:   r8   re   zmeta_addbmm.<locals>.<lambda>â  rm   r:   c                   ó   — yr„  r4   r4   r:   r8   re   zmeta_addbmm.<locals>.<lambda>ã  rm   r:   r   c                  óP   •— d‰ j                  d«      › d‰j                  d«      › �S )Nz8batch1 and batch2 must have same number of batches, got r   r¡   r¢   ©r†  r‡  s   €€r8   re   zmeta_addbmm.<locals>.<lambda>æ  s.   ø€ ÐJÈ6Ï;É;ÐWXË>ÐJZÐZ_Ð`f×`kÑ`kÐlmÓ`nÐ_oÐp€ r:   c            
      óš   •— d‰ j                  d«      › d‰ j                  d«      › d‰j                  d«      › d‰j                  d«      › d�	S )Nz#Incompatible matrix sizes for bmm (r3   r=   rý   r¡   ry   r¢   r"  s   €€r8   re   zmeta_addbmm.<locals>.<lambda>ê  sQ   ø€ Ø1°&·+±+¸a³.Ð1AÀÀ6Ç;Á;ÈqÃ>ÐBRð SØ—;‘;˜q“>Ð" ! F§K¡K°£NÐ#3°1ð6ð r:   c                   ó   — y)Nz.self tensor does not match matmul output shaper4   r4   r:   r8   re   zmeta_addbmm.<locals>.<lambda>ñ  rm   r:   )r£   r[  rV   rg   r†   r˜   )rš   r†  r‡  rˆ  r‡  r’  r“  s    ``    r8   Úmeta_addbmmr%  Ü  s  ù€ ð �;‰;�q‹>€DØ�;‰;�q‹>€DØ�;‰;˜˜d�|Ó$€DÜ	‡L�L�—‘“ Ñ"Ñ$HÔIÜ	‡L�L�—‘“ Ñ"Ñ$HÔIÜ	‡L�LØ�‰�A‹˜&Ÿ+™+ a›.Ñ(Üpôô 
‡L�LØ�‰�A‹˜&Ÿ+™+ a›.Ñ(ô	
ôô 
‡L�LØ�	‰	�!‹˜ÑÒ5 §¡¨1£°Ñ!5Ù@ôð �>‰>˜$Ÿ)™)›+Ó&Ð&r:   c                 ó@   — | j                  | j                  «       «      S rD   ©r˜   r£   )rš   r  Úkwargss      r8   Úmeta_randint_liker)  ö  s   € à�>‰>˜$Ÿ)™)›+Ó&Ð&r:   )Ú
grad_scaleÚ	found_infc       	         ón   ‡— | |||||fD ])  Št        j                  t        ‰t        «      ˆfd„«       Œ+ y )Nc                  ó    •— dt        ‰ «      › �S ©Nz'exponent must be a tensor list but got ©rz   ©Úls   €r8   re   z#meta__fused_adam_.<locals>.<lambda>  ó   ø€ Ð=¼dÀ1»g¸YÐG€ r:   ©rV   rg   rp   rÄ   )rš   ÚgradsÚexp_avgsÚexp_avg_sqsÚmax_exp_avg_sqsÚstate_stepsÚlrÚbeta1Úbeta2Úweight_decayÚepsÚamsgradÚmaximizer*  r+  r1  s                  @r8   Úmeta__fused_adam_r@  û  s:   ø€ ð& �E˜8 [°/À;ÐOò 
ˆÜ�‰Ü�qœ$ÓÛGõ	
ñ
r:   c       	         ó¼   ‡— | |||||fD ])  Št        j                  t        ‰t        «      ˆfd„«       Œ+ d„ } || «       ||«       ||«       ||«       ||«      fS )Nc                  ó    •— dt        ‰ «      › �S r.  r/  r0  s   €r8   re   z"meta__fused_adam.<locals>.<lambda>+  r2  r:   c                 óR   — | D �cg c]  }t        j                  |«      ‘Œ c}S c c}w rD   rJ  )Útensor_listr  s     r8   Úempty_like_listz)meta__fused_adam.<locals>.empty_like_list.  s!   € Ø-8Ö9¨”× Ñ  Õ#Ò9Ð9ùÒ9s   …$r3  )rš   r4  r5  r6  r7  r8  r9  r:  r;  r<  r=  r>  r?  r*  r+  rE  r1  s                   @r8   Úmeta__fused_adamrF    ss   ø€ ð& �E˜8 [°/À;ÐOò 
ˆÜ�‰Ü�qœ$ÓÛGõ	
ð
ò:ñ 	˜ÓÙ˜ÓÙ˜Ó!Ù˜Ó$Ù˜Ó(ðð r:   c                 óŠ  ‡ ‡— t        j                  ‰ j                  «       dk(  d„ «       t        j                  ‰j                  «       dk(  d„ «       t        j                  ‰ j                  t         j                  t         j
                  fv ˆ fd„«       t        j                  ‰j                  t         j                  u ˆfd„«       t        j                  ‰ j                  d«      ‰j                  d«      k(  ˆ ˆfd„«       ‰ j                  ‰ j                  d«      ‰j                  d«      ft         j                  ¬	«      S )
Nrý   c                   ó   — y)Nza must be a 2D tensorr4   r4   r:   r8   re   zmeta__int_mm.<locals>.<lambda>=  rm   r:   c                   ó   — y)Nzb must be a 2D tensorr4   r4   r:   r8   re   zmeta__int_mm.<locals>.<lambda>>  rm   r:   c                  ó"   •— d‰ j                   › �S )Nz'expected self to be int8 or uint8, got r‘   )r6   s   €r8   re   zmeta__int_mm.<locals>.<lambda>A  s   ø€ Ð9¸!¿'¹'¸ÐC€ r:   c                  ó"   •— d‰ j                   › �S )Nzexpected mat2 to be int8, got r‘   )r7   s   €r8   re   zmeta__int_mm.<locals>.<lambda>E  s   ø€ Ð0°·±°	Ð:€ r:   r3   r   c            
      óš   •— d‰ j                  d«      › d‰ j                  d«      › d‰j                  d«      › d‰j                  d«      › d�	S )Nz'Incompatible matrix sizes for _int_mm (r   r=   r3   r¡   ry   r¢   r5   s   €€r8   re   zmeta__int_mm.<locals>.<lambda>I  sM   ø€ Ø5°a·f±f¸Q³i°[ÀÀ!Ç&Á&ÈÃ)Àð MØ—6‘6˜!“9�+˜Q˜qŸv™v a›y˜k¨ð,ð r:   r‘   )	rV   rg   r†   r^   rt  r  r£   r˜   ru  r5   s   ``r8   Úmeta__int_mmrM  :  sÙ   ù€ ô 
‡L�L�—‘“˜A‘Ñ>Ô?Ü	‡L�L�—‘“˜A‘Ñ>Ô?Ü	‡L�LØ	�‰”E—J‘J¤§¡Ð,Ð,ÛCôô 
‡L�LØ	�‰”5—:‘:ÐÛ:ôô 
‡L�LØ	�‰ˆq‹	�Q—V‘V˜A“YÑô	
ôð �;‰;˜Ÿ™˜q›	 1§6¡6¨!£9Ð-´U·[±[ˆ;ÓAÐAr:   c                 óf  ‡ — t        j                  ‰ j                  «       dk(  d„ «       t        j                  ‰ j                  t         j                  u ˆ fd„«       ‰ j                  d«      }‰ j                  d«      dz  }‰ j                  |dz  ||dz  z  d|dz  ft         j                  ¬	«      S )
Nrý   c                   ó   — y©Nzw must be a 2D tensorr4   r4   r:   r8   re   z2meta__convert_weight_to_int4pack.<locals>.<lambda>S  rm   r:   c                  ó"   •— d‰ j                   › �S rC  r‘   r?  s   €r8   re   z2meta__convert_weight_to_int4pack.<locals>.<lambda>V  ó   ø€ Ð.¨q¯w©w¨iÐ8€ r:   r   r3   é   é   é    r‘   )rV   rg   r†   r^   r  r£   r˜   ru  ©r  Úinner_k_tilesr  rw  s   `   r8   Ú meta__convert_weight_to_int4packrX  Q  sž   ø€ ä	‡L�L�—‘“˜A‘Ñ>Ô?Ü	‡L�LØ	�‰”5—;‘;ÐÛ8ôð 	
�‰ˆq‹	€AØ	�‰ˆq‹	�A‰€AØ�;‰;à�‰FØ�- "Ñ$Ñ%ØØ˜QÑð		
ô �k‰kð ó ð r:   c                 óJ  ‡ — t        j                  ‰ j                  «       dk(  d„ «       t        j                  ‰ j                  t         j                  u ˆ fd„«       ‰ j                  d«      }‰ j                  d«      }‰ j                  ||dz  ft         j                  ¬«      S )Nrý   c                   ó   — yrP  r4   r4   r:   r8   re   z:meta__convert_weight_to_int4pack_for_cpu.<locals>.<lambda>g  rm   r:   c                  ó"   •— d‰ j                   › �S ©Nzexpected w to be int32, got r‘   r?  s   €r8   re   z:meta__convert_weight_to_int4pack_for_cpu.<locals>.<lambda>j  rR  r:   r   r3   r‘   )rV   rg   r†   r^   ru  r£   r˜   r  rV  s   `   r8   Ú(meta__convert_weight_to_int4pack_for_cpur]  e  s   ø€ ä	‡L�L�—‘“˜A‘Ñ>Ô?Ü	‡L�LØ	�‰”5—;‘;ÐÛ8ôð 	
�‰ˆq‹	€AØ	�‰ˆq‹	€AØ�;‰;Ø	
ˆA�‰FˆÜ�k‰kð ó ð r:   c                 óÆ  ‡ ‡‡— t        j                  ‰ j                  «       dk(  d„ «       ‰j                  j                  dk(  rdndŠt        j                  ‰j                  «       ‰k(  ˆfd„«       t        j                  ‰ j
                  t         j                  t         j                  t         j                  fv ˆ fd„«       t        j                  ‰j
                  t         j                  u ˆfd„«       ‰j                  j                  dk(  r‰j                  d«      n‰j                  d«      d	z  }‰ j                  ‰ j                  d«      |‰ j
                  ¬
«      S )Nrý   c                   ó   — y©Nzx must be a 2D tensorr4   r4   r:   r8   re   z*meta__weight_int4pack_mm.<locals>.<lambda>v  rm   r:   r   r5  c                  ó   •— d‰ › d�S )Nzw must be a zD tensorr4   )Úexpected_dims   €r8   re   z*meta__weight_int4pack_mm.<locals>.<lambda>x  s   ø€ °LÀÀÈhÐ2W€ r:   c                  ó"   •— d‰ j                   › �S rA  r‘   r<  s   €r8   re   z*meta__weight_int4pack_mm.<locals>.<lambda>{  ó   ø€ Ð5°a·g±g°YÐ?€ r:   c                  ó"   •— d‰ j                   › �S r\  r‘   r?  s   €r8   re   z*meta__weight_int4pack_mm.<locals>.<lambda>  rR  r:   r   rS  r‘   )rV   rg   r†   rÃ  rz   r^   r’  r“  r”  ru  r£   r˜   )r=   r  rE  rG  Údim_nrb  s   ``   @r8   Úmeta__weight_int4pack_mmrg  t  sì   ú€ ä	‡L�L�—‘“˜A‘Ñ>Ô?ØŸ™×*Ñ*¨eÒ3‘1¸€LÜ	‡L�L�—‘“˜LÑ(Ó*WÔXÜ	‡L�LØ	�‰”E—M‘M¤5§=¡=´%·.±.ÐAÐAÛ?ôô 
‡L�LØ	�‰”5—;‘;ÐÛ8ôð Ÿ™×+Ñ+¨uÒ4ˆA�F‰F�1ŒI¸!¿&¹&À»)Àa¹-€EØ�;‰;�q—v‘v˜a“y %¨q¯w©wˆ;Ó7Ð7r:   c                 ó(  ‡ ‡— t        j                  ‰ j                  «       dk(  d„ «       t        j                  ‰j                  «       dk(  d„ «       t        j                  ‰ j                  t         j                  t         j
                  t         j                  fv ˆ fd„«       t        j                  ‰j                  t         j                  u ˆfd„«       ‰ j                  ‰ j                  d«      ‰j                  d«      ‰ j                  ¬«      S )Nrý   c                   ó   — yr`  r4   r4   r:   r8   re   z2meta__weight_int4pack_mm_for_cpu.<locals>.<lambda>‡  rm   r:   c                   ó   — yrP  r4   r4   r:   r8   re   z2meta__weight_int4pack_mm_for_cpu.<locals>.<lambda>ˆ  rm   r:   c                  ó"   •— d‰ j                   › �S rA  r‘   r<  s   €r8   re   z2meta__weight_int4pack_mm_for_cpu.<locals>.<lambda>‹  rd  r:   c                  ó"   •— d‰ j                   › �S rC  r‘   r?  s   €r8   re   z2meta__weight_int4pack_mm_for_cpu.<locals>.<lambda>�  rR  r:   r   r‘   )
rV   rg   r†   r^   r’  r“  r”  r  r˜   r£   rH  s   ``  r8   Ú meta__weight_int4pack_mm_for_cpurm  …  ó¯   ù€ ä	‡L�L�—‘“˜A‘Ñ>Ô?Ü	‡L�L�—‘“˜A‘Ñ>Ô?Ü	‡L�LØ	�‰”E—M‘M¤5§=¡=´%·.±.ÐAÐAÛ?ôô 
‡L�LØ	�‰”5—;‘;ÐÛ8ôð �;‰;�q—v‘v˜a“y !§&¡&¨£)°1·7±7ˆ;Ó;Ð;r:   c                 ó(  ‡ ‡— t        j                  ‰ j                  «       dk(  d„ «       t        j                  ‰j                  «       dk(  d„ «       t        j                  ‰ j                  t         j                  t         j
                  t         j                  fv ˆ fd„«       t        j                  ‰j                  t         j                  u ˆfd„«       ‰ j                  ‰ j                  d«      ‰j                  d«      ‰ j                  ¬«      S )Nrý   c                   ó   — yr`  r4   r4   r:   r8   re   z;_weight_int4pack_mm_with_scales_and_zeros.<locals>.<lambda>–  rm   r:   c                   ó   — yrP  r4   r4   r:   r8   re   z;_weight_int4pack_mm_with_scales_and_zeros.<locals>.<lambda>—  rm   r:   c                  ó"   •— d‰ j                   › �S rA  r‘   r<  s   €r8   re   z;_weight_int4pack_mm_with_scales_and_zeros.<locals>.<lambda>š  rd  r:   c                  ó"   •— d‰ j                   › �S r\  r‘   r?  s   €r8   re   z;_weight_int4pack_mm_with_scales_and_zeros.<locals>.<lambda>ž  rR  r:   r   r‘   )
rV   rg   r†   r^   r’  r“  r”  ru  r˜   r£   )r=   r  rE  ÚqScaleÚqZeross   ``   r8   Ú)_weight_int4pack_mm_with_scales_and_zerosrv  ”  rn  r:   r6   r7   c                 ó   — | |z   dz
  |z  |z  S r2   r4   r5   s     r8   Úkai_rounduprx  £  s   € Ø�‰U�Q‰Y˜1Ñ Ñ!Ð!r:   c                 ó  ‡	‡
‡‡‡‡‡‡‡‡‡— | dk(  ry||k(  r(d}d}d}dŠdŠdŠd„ Šˆˆˆˆfd„Šˆfd„} ||||||«      S |dz  d	k(  rC||z  d	k(  r:d}d}d}dŠdŠdŠdŠ	ˆ	ˆˆfd
„}ˆ	ˆ
ˆˆˆˆˆfd„Šd„ Š
ˆ	fd„Šˆ	fd„Š |||||||«      S y y y )Nr5  rS  rT  rý   c                 ó8   — t        ||z  d«      }t        | |«      S )Nr5  ©rx  )rw  ÚkrÚsrÚkr_sr_roundedup4s       r8   Úkai_k_roundedupz3get_kai_packed_weight_size.<locals>.kai_k_roundedup³  s#   € ô $/¨r°B©w¸Ó#:Ð Ü" 1Ð&6Ó7Ð7r:   c                 óf   •—  ‰| ||«      }|dz  dk7  rt        d|› �«      ‚||dz  ‰z   ‰z   ‰z   z  S )Nrý   r   zk_internal must be even, got r  )	rw  Únrr|  r}  Ú
k_internalr  Úkai_num_bytes_biasÚkai_num_bytes_multiplier_rhsÚkai_num_bytes_sum_rhss	        €€€€r8   Ú9kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0z]get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0¹  s`   ø€ ñ -¨Q°°BÓ7�
à ‘N qÒ(Ü(Ð+HÈÈÐ)UÓVÐVàØ 1‘_Ø2ñ3à+ñ,ð )ñ)ñð r:   c                 ó>   •— t        | |«      |z  }| ‰||||«      z  S rD   r{  )r  rw  r�  r|  r}  Únum_rowsr†  s         €r8   Ú7kai_get_rhs_packed_size_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0z[get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_size_rhs_pack_nxk_qsi4cxp_qsu4cxs1s0È  s6   ø€ ô ' q¨"Ó-°Ñ3�ð ÙOØ˜2˜r 2óñðr:   rU  r   c                 óÜ   •— ||z  dk7  rt        d|› d|› d�«      ‚|‰	z  dk7  rt        d|› d‰	› d�«      ‚|‰z  dk7  rt        d|› d‰› d�«      ‚t        | |«      |z  }| ‰|||||«      z  S ©Nr   úbl (z) must be divisible by kr (ry   znr (z+) must be divisible by kai_nr_multiple_of (ú+) must be divisible by kai_bl_multiple_of (©rŠ   rx  )
r  rw  r�  r|  r}  Úblrˆ  Úkai_bl_multiple_ofÚ;kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0Úkai_nr_multiple_ofs
          €€€r8   Ú9kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0z]get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0à  sÅ   ø€ ð ˜‘G ’>Ü(¨4°¨tÐ3NÈrÈdÐRSÐ)TÓUÐUØÐ+Ñ+°Ò1Ü(Ø˜r˜dÐ"MÐN`ÐMaÐabÐcóð ð Ð+Ñ+°Ò1Ü(Ø˜r˜dÐ"MÐN`ÐMaÐabÐcóð ô ' q¨"Ó-°Ñ3�ð ÙQØ˜2˜r 2 róñðr:   c                 óî   •— ||z  dk7  rt        d|› d|› d�«      ‚|‰
z  dk7  rt        d|› d‰
› d�«      ‚|‰z  dk7  rt        d|› d‰› d�«      ‚ ‰	«       } ‰| |«      } ‰||«      }|||z  ‰z   ‰z   z  S r‹  r  )rw  r�  r|  r}  r�  Únum_bytes_multiplier_rhsÚnum_blocks_per_rowÚnum_bytes_per_blockr�  Ú#kai_get_bf16_datatype_size_in_bytesr’  Úkai_num_blocks_per_rowrƒ  Úkai_num_bytes_per_blockr…  s           €€€€€€€r8   r‘  z_get_kai_packed_weight_size.<locals>.kai_get_rhs_packed_stride_rhs_pack_nxk_qsi4c32p_qsu4c32s1s0÷  sâ   ø€ ð ˜‘G ’>Ü(¨4°¨tÐ3NÈrÈdÐRSÐ)TÓUÐUØÐ+Ñ+°Ò1Ü(Ø˜r˜dÐ"MÐN`ÐMaÐabÐcóð ð Ð+Ñ+°Ò1Ü(Ø˜r˜dÐ"MÐN`ÐMaÐabÐcóð ñ
 ,OÓ+PÐ(Ù%;¸A¸rÓ%BÐ"Ù&=ØÐ0ó'Ð#ð Ø(Ð+=Ñ=Ø+ñ,à(ñ)ñð r:   c                   ó   — y)Nrý   r4   r4   r:   r8   r˜  zGget_kai_packed_weight_size.<locals>.kai_get_bf16_datatype_size_in_bytes  s   € Ør:   c                 óV   •— |‰z  dk7  rt        d|› d‰› d�«      ‚t        | |«      |z  S )Nr   rŒ  r�  ry   rŽ  )rw  r�  r�  s     €r8   r™  z:get_kai_packed_weight_size.<locals>.kai_num_blocks_per_row  sH   ø€ ØÐ+Ñ+°Ò1Ü(Ø˜r˜dÐ"MÐN`ÐMaÐabÐcóð ô # 1 bÓ)¨RÑ/Ð/r:   c                 óH   •— | ‰z  dk7  rt        d| › d‰› d�«      ‚| dz  |z   S )Nr   rŒ  r�  ry   rý   r  )r�  r•  r�  s     €r8   rš  z;get_kai_packed_weight_size.<locals>.kai_num_bytes_per_block  sF   ø€ ØÐ+Ñ+°Ò1Ü(Ø˜r˜dÐ"MÐN`ÐMaÐabÐcóð ð ˜a™Ð#;Ñ;Ð;r:   r4   )Ún_bitsr¹  ÚKÚ	groupsizeÚkai_nrÚkai_krÚkai_srr‰  r“  r�  r˜  r‘  r†  r  r’  r™  rƒ  r„  rš  r…  s            @@@@@@@@@@@r8   Úget_kai_packed_weight_sizer¤  §  sç   ÿú€ Ø�‚{à˜Š>àˆFØˆFØˆFØ$%Ð!Ø+,Ð(Ø!"Ðò8÷ô
ñ KØ�1�f˜f fóð ð ˜‰^˜qÒ  Q¨¡]°aÒ%7ØˆFØˆFØˆFØ$%Ð!Ø!"ÐØ!"ÐØ!#Ðö÷.ò ò:ô0ô<ñ MØ�1�f˜f f¨ióð ð] &8Ð ð_ r:   c                 ó€  ‡ — t        j                  ‰ j                  t         j                  u ˆ fd„«       t         j                  j
                  j                  «       rŒ||k(  r|j                  t         j                  k(  s2||k  re|dz  dk(  r]||z  dk(  rU|j                  t         j                  k(  r8t        d|||«      }‰ j                  t        |«      t         j                  ¬«      S ‰ j                  «       |j                  «       z   }|�||j                  «       z  }‰ j                  |t         j                  ¬«      S )Nc                  ó"   •— d‰ j                   › �S rC  r‘   )Úweightss   €r8   re   z2meta__dyn_quant_pack_4bit_weight.<locals>.<lambda>0  s   ø€ Ð.¨w¯}©}¨oÐ>€ r:   rU  r   r5  r‘   )rV   rg   r^   r  rñ   Úkleidiairó   rZ   r”  r¤  r˜   rã   r—   )r§  Úscales_zerosrm  Ú
block_sizeÚin_featuresÚout_featuresÚpacked_weight_sizes   `      r8   Ú meta__dyn_quant_pack_4bit_weightr®  *  s  ø€ ô 
‡L�LØ�‰œŸ™Ð$Û>ôô ‡~�~×Ñ×+Ñ+Ô-Ø	�{Ò	" |×'9Ñ'9¼U¿[¹[Ò'Hà˜Ò$Ø˜R‘ 1Ò$Ø˜jÑ(¨AÒ-Ø×"Ñ"¤e§n¡nÒ4ô 8Øˆ|˜[¨*ó
Ðð × Ñ ¤Ð%7Ó!8ÄÇÁÐ ÓLÐLØ Ÿ™›¨<×+=Ñ+=Ó+?Ñ?ÐØÐØ˜dŸj™j›lÑ*ÐØ×ÑÐ/´u·{±{ÐÓCÐCr:   c                 ón  ‡ ‡‡— t        j                  ‰ j                  «       dk(  d„ «       t        j                  ‰ j                  t         j                  k(  xs$ ‰ j                  t         j
                  k(  xr ‰‰k(  ˆˆˆ fd„«       ‰ j                  d«      }‰ j                  ||‰ j                  ¬«      S )Nrý   c                   ó   — y)Nzinput must be a 2D tensorr4   r4   r:   r8   re   z-meta__dyn_quant_matmul_4bit.<locals>.<lambda>M  rm   r:   c                  ó.   •— d‰j                   › d‰ › d‰› �S )NzPexpected input to be f32 or bf16 (bf16 requires block_size == in_features), got z with block_size=z and in_features=r‘   )rª  r«  Úinps   €€€r8   re   z-meta__dyn_quant_matmul_4bit.<locals>.<lambda>Q  s*   ø€ ðØ—9‘9�+Ð.¨z¨lÐ:KÈKÈ=ðZð r:   r   r‘   )rV   rg   r†   r^   r’  r”  r£   r˜   )r²  Úpacked_weightsrª  r«  r¬  rÒ  s   ` ``  r8   Úmeta__dyn_quant_matmul_4bitr´  E  s‡   ú€ ô 
‡L�L�—‘“˜a‘Ñ!DÔEÜ	‡L�LØ	�‰”e—m‘mÑ	#ò 	GØ�I‰IœŸ™Ñ'ÒE¨J¸+Ñ,Eõ	
ôð 	�‰�‹€AØ�=‰=˜˜L°·	±	ˆ=Ó:Ð:r:   c                 ó(  ‡ ‡— t        j                  ‰ j                  «       dk(  d„ «       t        j                  ‰ j                  t         j                  t         j
                  t         j                  fv ˆ fd„«       t        j                  ‰j                  «       dk(  d„ «       t        j                  ‰j                  t         j                  u ˆfd„«       ‰ j                  ‰ j                  d«      ‰j                  d«      ‰ j                  ¬«      S )Nrý   c                   ó   — yr`  r4   r4   r:   r8   re   z*meta__weight_int8pack_mm.<locals>.<lambda>\  rm   r:   c                  ó"   •— d‰ j                   › �S rA  r‘   r<  s   €r8   re   z*meta__weight_int8pack_mm.<locals>.<lambda>_  rd  r:   c                   ó   — yrP  r4   r4   r:   r8   re   z*meta__weight_int8pack_mm.<locals>.<lambda>a  rm   r:   c                  ó"   •— d‰ j                   › �S )Nzexpected w to be int8, got r‘   r?  s   €r8   re   z*meta__weight_int8pack_mm.<locals>.<lambda>d  s   ø€ Ð-¨a¯g©g¨YÐ7€ r:   r   r‘   )
rV   rg   r†   r^   r’  r“  r”  rt  r˜   r£   )r=   r  Úq_scaless   `` r8   Úmeta__weight_int8pack_mmr»  Z  s¯   ù€ ä	‡L�L�—‘“˜A‘Ñ>Ô?Ü	‡L�LØ	�‰”E—M‘M¤5§=¡=´%·.±.ÐAÐAÛ?ôô 
‡L�L�—‘“˜A‘Ñ>Ô?Ü	‡L�LØ	�‰”5—:‘:ÐÛ7ôð �;‰;�q—v‘v˜a“y !§&¡&¨£)°1·7±7ˆ;Ó;Ð;r:   c                 ón  ‡ ‡‡— t        j                  ‰ j                  «       dk\  ˆ fd„«       t        j                  ‰j                  «       dk\  ˆfd„«       t        j                  ‰ j                  d«      ‰j                  d«      k(  ˆ ˆfd„«       t        j                  t	        j
                  ‰ j                  «      ˆ fd„«       t        j                  t	        j
                  ‰j                  «      ˆfd„«       t        j                  |dk\  d	„ «       t        j                  ‰d
v ˆfd„«       ‰ j                  d«      }‰j                  d«      }‰ j                  d d }‰j                  d d }t        t        j                  ||«      «      }|j                  ||g«       ‰ j                  |«      S )Nrý   c                  ó,   •— d‰ j                  «       › d�S )Nz1cdist only supports at least 2D tensors, X1 got: r  r�   ©Úx1s   €r8   re   z$meta_cdist_forward.<locals>.<lambda>m  ó   ø€ ÐCÀBÇFÁFÃHÀ:ÈQÐO€ r:   c                  ó,   •— d‰ j                  «       › d�S )Nz1cdist only supports at least 2D tensors, X2 got: r  r�   ©Úx2s   €r8   re   z$meta_cdist_forward.<locals>.<lambda>q  rÀ  r:   rœ   c                  óP   •— d‰ j                  d«      › d‰j                  d«      › �S )Nz4X1 and X2 must have the same number of columns. X1: rœ   z X2: r¢   )r¿  rÃ  s   €€r8   re   z$meta_cdist_forward.<locals>.<lambda>u  s,   ø€ ÐFÀrÇwÁwÈrÃ{ÀmÐSXÐY[×Y`ÑY`ÐacÓYdÐXeÐf€ r:   c                  ó"   •— d‰ j                   › �S )Nz3cdist only supports floating-point dtypes, X1 got: r‘   r¾  s   €r8   re   z$meta_cdist_forward.<locals>.<lambda>y  ó   ø€ ÐEÀbÇhÁhÀZÐP€ r:   c                  ó"   •— d‰ j                   › �S )Nz3cdist only supports floating-point dtypes, X2 got: r‘   rÂ  s   €r8   re   z$meta_cdist_forward.<locals>.<lambda>}  rÆ  r:   r   c                   ó   — y)Nz)cdist only supports non-negative p valuesr4   r4   r:   r8   re   z$meta_cdist_forward.<locals>.<lambda>  rm   r:   )Nr   r3   rý   c                  ó   •— d‰ › �S )Nz(possible modes: None, 0, 1, 2, but was: r4   )Úcompute_modes   €r8   re   z$meta_cdist_forward.<locals>.<lambda>‚  s   ø€ Ð:¸<¸.ÐI€ r:   rò  )rV   rg   r†   r£   rO   Úis_float_dtyper^   r™   rÄ   Úbroadcast_shapesÚextendr˜   )	r¿  rÃ  rt  rÊ  Úr1Úr2Úbatch_tensor1Úbatch_tensor2r™  s	   `` `     r8   Úmeta_cdist_forwardrÒ  i  sJ  ú€ ä	‡L�LØ
�‰‹�A‰ÛOôô 
‡L�LØ
�‰‹�A‰ÛOôô 
‡L�LØ
�‰�‹�r—w‘w˜r“{Ñ"Üfôô 
‡L�LÜ×Ñ˜RŸX™XÓ&ÛPôô 
‡L�LÜ×Ñ˜RŸX™XÓ&ÛPôô 
‡L�L��a‘ÑLÔMÜ	‡L�LØ˜Ð'ÛIôð 
�‰�‹€BØ	�‰�‹€BØ—H‘H˜S˜b�M€MØ—H‘H˜S˜b�M€MÜœ×.Ñ.¨}¸mÓLÓM€LØ×Ñ˜˜R˜Ô!Ø�<‰<˜Ó%Ð%r:   c                 ó4  — |j                   d   }|j                   d   }|j                   d   }|j                   d d }|j                   d d }	t        t        j                  ||	«      «      }
|
j	                  «       }|j                  ||g«       t        j                  |
«      }|dk(  s|dk(  s
|dk(  s|dk(  rt        j                  |«      S |t        |j                   «      k7  r|j                  |«      }t        j                  |t        j                  ¬«      S )Nrœ   rò  r   r«   )r™   rÄ   rV   rÌ  ÚcopyrÍ  ÚmathÚprodÚ
zeros_liker[  r¯   r°   )rx  r¿  rÃ  rt  ÚcdistÚc1rÎ  rÏ  rÐ  rÑ  r®  Útensor1_expand_sizeÚbatch_products                r8   Úmeta_cdist_backwardrÜ  �  sô   € ð 
�‰�"‰€BØ	�‰�"‰€BØ	�‰�"‰€BØ—H‘H˜S˜b�M€MØ—H‘H˜S˜b�M€MÜ¤× 6Ñ 6°}ÀmÓ TÓUÐØ.×3Ñ3Ó5ÐØ×Ñ  B˜xÔ(Ü—I‘IÐ2Ó3€MØ	ˆQ‚w�"˜’'˜R 1šW¨¸Ò(:Ü×Ñ Ó#Ð#Øœd 2§8¡8›nÒ,Ø�Y‰YÐ*Ó+ˆÜ×Ñ˜B¬e×.EÑ.EÔFÐFr:   c	                 ó  ‡ ‡‡‡‡‡— t        j                  ‰j                  t         j                  t         j                  fv ˆfd„«       t        j                  ‰j                  t         j                  t         j                  fv ˆfd„«       t        j                  t        j                  ‰ j                  «      ˆ fd„«       ‰j                  d«      }	|rt        j                  |	dk\  d„ «       |	dz  }	‰ j                  |	‰ j                  d«      «      }
‰�}t        j                  |t        k(  d„ «       t        j                  ‰j                  dk(  ˆfd„«       t        j                  ‰j                  «       ‰j                  «       k(  ˆˆfd	„«       ˆfd
„Šd„ Šˆˆfd„}t        ‰«      dk7  r|‰j                  ‰j                  d«      «      }‰j                  ‰j                  «       «      }|t        k(  r"‰j                  |	‰ j                  d«      «      }nè‰j                  d«      }nÖ |‰ ‰|
|«      }|t        t        fv s|s!‰j                  ‰j                  d«      «      }n‰j                  d«      }‰j                  |	«      }‰j                  d   }|t        k(  rA|rt        j                  |dk\  d„ «       |dz  }‰j                  |‰ j                  d   «      }n‰j                  |j                  «       «      }|
|||fS )Nc                  ó"   •— d‰ j                   › �S )Nz(expected indices to be long or int, got r‘   )r´   s   €r8   re   z$meta_embedding_bag.<locals>.<lambda>²  ó   ø€ Ð:¸7¿=¹=¸/ÐJ€ r:   c                  ó"   •— d‰ j                   › �S )Nz(expected offsets to be long or int, got r‘   )r©  s   €r8   re   z$meta_embedding_bag.<locals>.<lambda>¶  rß  r:   c                  ó"   •— d‰ j                   › �S )Nz/expected weight to be floating point type, got r‘   )rk  s   €r8   re   z$meta_embedding_bag.<locals>.<lambda>º  s   ø€ ÐAÀ&Ç,Á,ÀÐP€ r:   r   r3   c                   ó   — y©Nz1include_last_offset: numBags should be at least 1r4   r4   r:   r8   re   z$meta_embedding_bag.<locals>.<lambda>Á  rm   r:   c                   ó   — y)Nz@embedding_bag: per_sample_weights only supported with mode='sum'r4   r4   r:   r8   re   z$meta_embedding_bag.<locals>.<lambda>Ê  rm   r:   c                  ó$   •— d‰ j                   › d�S )Nz1expected per_sample_weights to be 1D tensor, got r  r7  )Úper_sample_weightss   €r8   re   z$meta_embedding_bag.<locals>.<lambda>Î  s   ø€ ÐGÐHZ×H_ÑH_ÐG`Ð`aÐb€ r:   c                  óN   •— d‰j                  «       › d‰ j                  «       › d�S )Nz%expected per_sample_weights.numel() (z$ to be the same as indices.numel() (ry   ©r—   )r´   ræ  s   €€r8   re   z$meta_embedding_bag.<locals>.<lambda>Ò  s4   ø€ Ø7Ð8J×8PÑ8PÓ8RÐ7Sð T6Ø6=·m±m³oÐ5FÀaðIð r:   c                 óD   •—  ‰| ||«      xr |j                  d«      dk(  S ©Nr   r3   ©rÆ   )r\  r©  r÷   Úpadding_idxÚis_fast_path_index_selects       €r8   Úis_fast_path_index_select_scalez;meta_embedding_bag.<locals>.is_fast_path_index_select_scaleØ  s(   ø€ á% c¨6°;Ó?ÒXÀEÇLÁLÐQRÃOÐWXÑDXð	
r:   c                 óà   — | j                   t        j                  k(  xs | j                   t        j                  k(  xr1 | j	                  d«      dk(  xr |j	                  d«      dk(  xr |dk  S ©Nr3   r   )r^   rV   rZ   rX   rÆ   )r\  r÷   rì  s      r8   rí  z5meta_embedding_bag.<locals>.is_fast_path_index_selectÝ  sb   € à�Y‰Yœ%Ÿ+™+Ñ%Ò@¨¯©´e·j±jÑ)@ò  Ø—
‘
˜1“ Ñ"ò à—‘˜aÓ  AÑ%ò ð ˜a‘ð		
r:   c                 ó2   •— |� ‰| |||«      S  ‰| ||«      S rD   r4   )r\  r©  r÷   rì  rí  rî  s       €€r8   Úis_fast_pathz(meta_embedding_bag.<locals>.is_fast_pathå  s)   ø€ ØÐÙ2°3¸¸vÀ{ÓSÐSá,¨S°&¸+ÓFÐFr:   rè   c                   ó   — yrã  r4   r4   r:   r8   re   z$meta_embedding_bag.<locals>.<lambda>ÿ  rm   r:   )rV   rg   r^   r•   rã   rO   rË  r£   r˜   ÚMODE_SUMr¥   r—   rð   ÚMODE_MAXÚ	MODE_MEANr™   )rk  r´   r©  Úscale_grad_by_freqrŒ  Úsparseræ  Úinclude_last_offsetrì  Únum_bagsr÷   rò  Ú
offset2bagÚbag_sizeÚmax_indicesÚfast_path_sumÚnumBagsrí  rî  s   ```   `          @@r8   Úmeta_embedding_bagr   ¤  s}  ý€ ô 
‡L�LØ�‰œ%Ÿ*™*¤e§i¡iÐ0Ð0ÛJôô 
‡L�LØ�‰œ%Ÿ*™*¤e§i¡iÐ0Ð0ÛJôô 
‡L�LÜ×Ñ˜VŸ\™\Ó*ÛPôð
 �|‰|˜A‹€HÙÜ�‰Ø˜‰MÙGô	
ð 	�A‰ˆà×Ñ˜h¨¯©°A«Ó7€FàÐ%Ü�‰Ø”HÑÙVô	
ô 	�‰Ø×#Ñ# qÑ(Ûbô	
ô 	�‰Ø×$Ñ$Ó&¨'¯-©-«/Ñ9ôô	
ô
ò

õGô �7Ó˜uÒ$Ø×&Ñ& w§|¡|°A£Ó7ˆ
Ø×$Ñ$ W§\¡\£^Ó4ˆØ”8ÒØ!×+Ñ+¨H°f·k±kÀ!³nÓE‰Kà!×+Ñ+¨AÓ.‰Ká$ VÐ-?ÀÈÓUˆØ”IœxÐ(Ñ(±Ø ×*Ñ*¨7¯<©<¸«?Ó;‰Jà ×*Ñ*¨1Ó-ˆJØ×$Ñ$ XÓ.ˆà—-‘- Ñ"ˆØ”8ÒÙ"Ü—‘Ø˜q‘LÙOôð ˜1‘�Ø!×+Ñ+¨G°V·\±\À!±_ÓE‰Kà!×+Ñ+¨H¯M©M«OÓ<ˆKØ�:˜x¨Ð4Ð4r:   c                 óŽ   — t        | ||g|¢­Ž \  }}}}t        |«      dk(  r|j                  |j                  «       «      }||||fS )Nrè   )r   rð   r˜   r£   )rk  r´   r©  rQ   r÷   rû  rü  rý  s           r8   Úmeta_embedding_bag_forward_onlyr    sX   € ä0BØ�˜ð1Ø#'ò1Ñ-€FˆJ˜ +ô �7Ó˜uÒ$Ø×$Ñ$ W§\¡\£^Ó4ˆØ�:˜x¨Ð4Ð4r:   c                 ó¶   — |r|S | j                   j                  s| j                   j                  r| j                   S |rt        j                  S | j                   S rD   )r^   r  rî   rV   r•   )rØ   r^   Úpromote_int_to_longs      r8   Ú_get_reduction_dtyper    sD   € áØˆà‡{�{×$Ò$¨¯©×(>Ò(>Ø�{‰{ÐÙ	Ü�z‰zÐà�;‰;Ðr:   r‘   c                óž   — t        | |d¬«      }t        j                  | j                  |«      }t	        | ||«      }| j                  ||¬«      S )NT)r  r‘   )r  rO   r¼  r™   r½  r˜   )rØ   r¾  r¿  r^   r  r™  s         r8   Úmeta_nansumr    sI   € ô (¨¨uÈ$ÔO€LÜ×Ñ §¡¨TÓ2€DÜ+¨E°4¸ÓA€LØ�?‰?˜<¨|ˆ?Ó<Ð<r:   c           	      ó¤   — t        j                  | j                  t        t	        | j                  «       «      «      «      }| j                  |«      S rD   )rO   r¿  r™   rf   rÅ   r†   r˜   )rØ   r™  s     r8   Úmeta_medianr	  (  s<   € ä×7Ñ7Ø�‰”Uœ5 §¡£Ó-Ó.ó€Lð �?‰?˜<Ó(Ð(r:   c                 ó  — t        | «      dk(  rt        j                  d«       t        j                  | j                  |f«      }t        | ||«      }| j                  |«      | j                  |t        j                  ¬«      fS )Nré   zmedian CUDA with indices outputr‘   )	rð   rO   Úalert_not_deterministicr¼  r™   r½  r˜   rV   r•   )rØ   r†   r¿  r™  s       r8   Úmeta_median_mode_dimr  0  sp   € ô �5Ó˜VÒ#Ü×%Ñ%Ð&GÔHÜ
×
Ñ
˜uŸ{™{¨S¨FÓ
3€CÜ+¨E°3¸Ó@€Là�‰˜Ó%Ø�‰˜¬E¯J©JˆÓ7ðð r:   c                 ó   — | S rD   r4   r±   s    r8   Úmeta_logical_not_r  F  r�  r:   c                 ó¶  ‡‡— t        j                  t        |«      | j                  «       k\  d„ «       t	        |«      D ]"  \  ŠŠt        j                  ‰dk\  ˆˆfd„«       Œ$ t        |«      | j                  «       z
  }d|z  t        | j                  «      z   }t        t        |«      «      D �cg c]  }||   ||   z  ‘Œ }}| j                  |«      S c c}w )Nc                   ó   — y)NzZNumber of dimensions of repeat dims can not be smaller than number of dimensions of tensorr4   r4   r:   r8   re   zmeta_repeat.<locals>.<lambda>O  rm   r:   r   c                  ó   •— d‰› d‰ › �S )Nz"Repeats cannot be negative, found rö  r4   )rÜ   Úreps   €€r8   re   zmeta_repeat.<locals>.<lambda>T  s   ø€ Ð8¸¸¸ZÈÀsÐK€ r:   r/  )	rV   rg   rÃ   r†   rÿ  rf   r™   rÅ   r˜   )rš   rß  Únum_new_dimensionsÚpadded_sizerÜ   Útarget_sizer  s       ` @r8   Úmeta_repeatr  K  sÂ   ù€ ä	‡L�LÜˆG‹˜Ÿ™›
Ñ"Ùlôô ˜GÓ$ò 
‰ˆˆ3Ü�‰Ø�1‰HÜKõ	
ð
ô ˜W›¨¯©«
Ñ2ÐØÐ+Ñ+¬e°D·J±JÓ.?Ñ?€KÜ8=¼cÀ'»lÓ8KÖL°1�;˜q‘> G¨A¡JÓ.ÐL€KÐLØ�>‰>˜+Ó&Ð&ùò Ms   Â1Cc                 ó   — | S rD   r4   r±   s    r8   Ú
meta_zero_r  _  r�  r:   c                 óz   — t        |t        j                  «      r t        | j                  |j                  «       | S rD   )rp   rV   r   ri   r™   ©rš   r¤   s     r8   Úmeta_binop_inplacer  d  s)   € ô �%œŸ™Ô&Ü §
¡
¨E¯K©KÔ8Ø€Kr:   c                 óø   — d„ }d„ }d„ } || «      r ||«      rt        d«      ‚ || «      r ||«      st        d«      ‚t        |t        j                  «      r t	        | j
                  |j
                  «       | S )a*  
    Some checks for inplace ops.
    Checks for promotion rules for some dtypes.
    int.add/sub_(float) and bool.add/sub_(others) are rejected.
    Promoting in these in-place operations would require reallocating
    and copying over elements, hence not allowed.
    Checks for alpha param.
    c                 ó€   — t        | t        «      rt        j                  | j                  «      S t        | t
        «      S rD   )rp   r   rO   rm  r^   r   ©rs   s    r8   Úis_integericz.meta_binop_inplace_alpha.<locals>.is_integeric‡  s.   € Ü�cœ:Ô&Ü×)Ñ)¨#¯)©)Ó4Ð4ä˜c¤7Ó+Ð+r:   c                 ó€   — t        | t        «      rt        j                  | j                  «      S t        | t
        «      S rD   )rp   r   rO   rË  r^   r   r  s    r8   Ú
is_floaticz,meta_binop_inplace_alpha.<locals>.is_floatic�  s.   € Ü�cœ:Ô&Ü×'Ñ'¨¯	©	Ó2Ð2ä˜c¤9Ó-Ð-r:   c                 ó€   — t        | t        «      rt        j                  | j                  «      S t        | t
        «      S rD   )rp   r   rO   Úis_boolean_dtyper^   r   r  s    r8   Úis_booleanicz.meta_binop_inplace_alpha.<locals>.is_booleanic“  s.   € Ü�cœ:Ô&Ü×)Ñ)¨#¯)©)Ó4Ð4ä˜c¤8Ó,Ð,r:   z]Promotion of int.add/sub_(float) in in-place ops are not possible due to element size change.z_Promotion of book.add/sub_(others) in in-place ops are not possible due to element size change.)rX  rp   rV   r   ri   r™   )rš   r¤   r‡  r  r!  r$  s         r8   Úmeta_binop_inplace_alphar%  u  sz   € ò$,ò.ò-ñ �DÔ™j¨Ô/ÜØkó
ð 	
ñ
 �DÔ¡,¨uÔ"5ÜØmó
ð 	
ô �%œŸ™Ô&Ü §
¡
¨E¯K©KÔ8Ø€Kr:   c                 ó:   — t        | |t        j                  ¬«      S ©Nrâ  ©rT   r   rP   ©rš   r¤   r‡  s      r8   Úmeta_binop_alphar*  ª  s   € ô ØˆeÔ$C×$KÑ$Kôð r:   c                 ó8   — t        | t        j                  ¬«      S r'  r(  )rš   r(  s     r8   Ú
meta_roundr,  ¶  s   € äØÔ<×DÑDôð r:   c                 ól  ‡ ‡‡— t        j                  t        j                  ‰j                  «      ˆ ˆfd„«       t        ‰t         j                  «      r8t        j                  t        j                  ‰j                  «      ˆ ˆfd„«       y t        j                  t        ‰t        «      ˆ ˆfd„«       y )Nc                  ó&   •— ‰ › d‰j                   › �S )Nz7: Expected input tensor to have an integral dtype. Got r‘   )r  rš   s   €€r8   re   z#shift_dtype_check.<locals>.<lambda>À  s   ø€ �7�)ÐRÐSW×S]ÑS]ÐR^Ð_€ r:   c                  ó&   •— ‰ › d‰j                   › �S )Nz6: Expected shift value to have an integral dtype. Got r‘   ©r  rÑ  s   €€r8   re   z#shift_dtype_check.<locals>.<lambda>Å  s   ø€ �w�iÐUÐVY×V_ÑV_ÐU`Ða€ r:   c                  ó   •— ‰ › d‰› �S )Nz): Expected shift value to be an int. Got r4   r0  s   €€r8   re   z#shift_dtype_check.<locals>.<lambda>Ê  s   ø€ �w�iÐHÈÈÐN€ r:   )rV   rg   rO   rm  r^   rp   r   r   )r  rš   rÑ  s   ```r8   Úshift_dtype_checkr2  ½  sp   ú€ Ü	‡L�LÜ×Ñ˜tŸz™zÓ*Ü_ôô �#”u—|‘|Ô$Ü�‰Ü×"Ñ" 3§9¡9Ó-Üaõ	
ô
 	�‰Ü�sœGÓ$ÜNõ	
r:   c                 óT   — t        d| |«       t        | |t        j                  ¬«      S )NÚrshiftrâ  ©r2  rT   r   rP   r  s     r8   Úmeta_rshiftsr6  Î  ó)   € ä�h  eÔ,ÜØˆeÔ$C×$KÑ$Kôð r:   c                 óT   — t        d| |«       t        | |t        j                  ¬«      S )NÚlshiftrâ  r5  r  s     r8   Úmeta_lshiftsr:  Ö  r7  r:   c                 ó8   — | j                  | j                  «      S rD   r  r±   s    r8   Ú	meta_zeror<  Þ  s   € à�>‰>˜$Ÿ*™*Ó%Ð%r:   c                 ó   — | S rD   r4   ©rš   rÑ  s     r8   Ú
meta_fill_r?  ã  r�  r:   c                 ó,   — t        j                  | «      S rD   rJ  r>  s     r8   Ú	meta_fillrA  è  ó   € ä×Ñ˜DÓ!Ð!r:   c                 ó   — | S rD   r4   r±   s    r8   Ú
meta_relu_rD  í  r�  r:   c                 ó:   — t        | |t        j                  ¬«      S r'  r(  r)  s      r8   Úmeta__add_relurF  ò  s   € ô ØˆeÔ$C×$KÑ$Kôð r:   c                 ó,   — t        j                  | «      S rD   rJ  ©rš   ÚnoiseÚlowerr  ræ  r
  s         r8   Úmeta_rrelu_with_noiserK  ú  s   € ô
 ×Ñ˜DÓ!Ð!r:   c                 óV   — t        j                  | «      t        j                  |«      fS rD   rJ  rH  s         r8   Ú meta_rrelu_with_noise_functionalrM    s%   € ô ×Ñ˜DÓ!¤5×#3Ñ#3°EÓ#:Ð:Ð:r:   c                 ó   — | S rD   r4   )rš   rJ  r  ræ  r
  s        r8   Úmeta_rrelu_with_noise_rO  	  s	   € ð €Kr:   c                 ó,   — t        j                  | «      S rD   rJ  ©rš   r´   r³   Ú
accumulates       r8   Úmeta_index_putrS    rB  r:   c                 óF   — t        | j                  |j                  «       | S rD   ©ri   r™   )rš   r±  Úvalues      r8   Úmeta_masked_fill_rW    s   € ä˜DŸJ™J¨¯
©
Ô3Ø€Kr:   c                 óŠ   — | j                  | j                  «       «      j                  t        j                  | «      ¬«      }|S r»   )r˜   r£   rY  rO   r   )rš   r±  r©  Úmasked_scales       r8   Úmeta__masked_scalerZ    s<   € à—>‘> $§)¡)£+Ó.×1Ñ1Ü×1Ñ1°$Ó7ð 2ó €Lð Ðr:   c                 óî   ‡ ‡— t        j                  |j                  t         j                  t         j                  fv d„ «       t        j                  ‰ j                  ‰j                  k(  ˆ ˆfd„«       ‰ S )Nc                   ó   — y)NzMask must be bool or uint8r4   r4   r:   r8   re   z&meta_masked_scatter_.<locals>.<lambda>&  rm   r:   c                  ó<   •— d‰ j                   › d‰j                   › �S )NzEmasked_scatter: expected self and source to have same dtypes but got r¡   r‘   )rš   rœ  s   €€r8   re   z&meta_masked_scatter_.<locals>.<lambda>*  s"   ø€ ð ØŸ*™*˜ U¨6¯<©<¨.ð:€ r:   )rV   rg   r^   r   r  )rš   r±  rœ  s   ` `r8   Úmeta_masked_scatter_r^  #  sU   ù€ ä	‡L�LØ�
‰
”u—z‘z¤5§;¡;Ð/Ð/Ñ1Uôô 
‡L�LØ�
‰
�f—l‘lÑ"ô	:ôð
 €Kr:   c                 ó„   — t        | |«      \  } }t        j                  | t        j                  ¬«      }t	        |||«      S r»   )r&   rV   r¯   r°   r^  )rš   r±  rœ  r÷   s       r8   Úmeta_masked_scatterr`  0  s;   € ô " $¨Ó-�J€Dˆ$Ü×Ñ˜d´%×2IÑ2IÔJ€FÜ ¨¨fÓ5Ð5r:   c                 ó$   — | j                  |«      S rD   r¶  )rš   r±  rv  s      r8   Úmeta_masked_scatter_backwardrb  8  s   € à�>‰>˜%Ó Ð r:   c                 ó   — | S rD   r4   rQ  s       r8   Úmeta_index_put_rd  =  r�  r:   c           	      ó  ‡ ‡‡‡‡‡‡— ddl m}m} t        j                  ‰ j                  «       dk(  d„ «       t        j                  ‰j                  «       dk(  d„ «       t        j                  ‰ j                  ‰j                  k(  ˆ ˆfd„«       ‰ j                  «       }‰j                  «       Š|d   Š|d   Š|d   }‰d   }	‰||	fŠt        j                   | |‰d   ‰«       |‰d   ‰«      «      ˆˆˆfd	„«       |rš‰ j                  t        j                  k(  xs ‰ j                  t        j                  k(  xr |t        j                  k(  }
t        j                  |‰ j                  k(  xs |
d
„ «       ‰j                  ‰«      j                  |«      }n‰j                  ‰«      }|sY‰�Wt        j                  ‰j                  «       dk(  d„ «       t        j                   |‰j                  «       ‰«      ˆˆfd„«       |S )Nr   )Úsym_andr€  r0   c                   ó   — yr‚  r4   r4   r:   r8   re   z)common_meta_baddbmm_bmm.<locals>.<lambda>E  rm   r:   c                   ó   — yr„  r4   r4   r:   r8   re   z)common_meta_baddbmm_bmm.<locals>.<lambda>F  rm   r:   c                  ó<   •— d‰ j                   › d‰j                   › �S )Nzexpected scalar type z but found r‘   r"  s   €€r8   re   z)common_meta_baddbmm_bmm.<locals>.<lambda>I  s   ø€ Ð'¨¯© ~°[ÀÇÁÀÐO€ r:   rý   r3   c            	      ó.   •— d‰› d‰› d‰ d   › d‰ d   › d�	S r‰  r4   r‹  s   €€€r8   re   z)common_meta_baddbmm_bmm.<locals>.<lambda>W  s5   ø€ ÐRÐSUÐRVØ
ÐÐ˜l¨<¸©?Ð*;¸2¸lÈ1¹oÐ=NÈbðR€ r:   c                   ó   — y)Nzfout_dtype only supported for torch.float32 output with float16/bfloat16 inputs or same as input dtypesr4   r4   r:   r8   re   z)common_meta_baddbmm_bmm.<locals>.<lambda>`  rm   r:   c                   ó   — y)Nzself must be a 3D tensorr4   r4   r:   r8   re   z)common_meta_baddbmm_bmm.<locals>.<lambda>h  rm   r:   c                  ó0   •— d‰ › d‰j                  «       › �S )Nz*Expected an input tensor shape with shape z but got shape: r¢   )rª  Úself_baddbmms   €€r8   re   z)common_meta_baddbmm_bmm.<locals>.<lambda>k  s!   ø€ Ð@ÀÀÐM]Ð^j×^oÑ^oÓ^qÐ]rÐs€ r:   )rV  rf  r€  rV   rg   r†   r^   r£   r“  r”  r’  r˜   rY  )r†  r‡  Úis_bmmrn  ro  rf  r€  r•  Úres_rowsÚres_colsÚsupported_out_dtyper÷   rŒ  r�  rŽ  rª  s   `` `        @@@@r8   Úcommon_meta_baddbmm_bmmrs  B  s²  þ€ ßEä	‡L�L�—‘“ Ñ"Ñ$HÔIÜ	‡L�L�—‘“ Ñ"Ñ$HÔIÜ	‡L�LØ�‰˜Ÿ™Ñ$ÜOôð
 —;‘;“=€LØ—;‘;“=€Là	�a‰€BØ# A‘ÐØ˜A‰€HØ˜A‰€HØ�x Ð*€Kä	‡L�LÙ‘�| A‘¨Ó+©V°LÀ±OÐEUÓ-VÓWõ	Rôñ
 à�L‰LœEŸM™MÑ)ÒK¨V¯\©\¼U¿^¹^Ñ-Kò)àœ5Ÿ=™=Ñ(ð 	ô 	�‰Ø˜Ÿ™Ñ%Ò<Ð)<Ù|ô	
ð ×!Ñ! +Ó.×1Ñ1°)Ó<‰ð ×!Ñ! +Ó.ˆá�lÐ.Ü�‰�\×%Ñ%Ó'¨1Ñ,Ñ.PÔQÜ�‰Ù�<×$Ñ$Ó&¨Ó4Üsô	
ð
 €Mr:   c                 ó   — t        | |d«      S )NT©rs  )rš   r|  s     r8   Úmeta_bmmrv  q  s   € ô # 4¨¨tÓ4Ð4r:   c                 ó    — t        | |d|¬«      S )NT)ro  ru  )rš   r|  ro  s      r8   Úmeta_bmm_dtyperx  w  s   € ô # 4¨¨t¸yÔIÐIr:   c                 óh   — | |z  }| |z  }|dk7  r"t        |dk  «      t        |dk  «      k7  r|dz  }|S rê  )r   )r=   r>   Úqr�  s       r8   Údiv_rtnr{  }  sB   € Ø	ˆQ‰€AØ	ˆA‰€Að 	ˆA‚v”4˜˜A™“;¤$ q¨1¡u£+Ò-Ø	ˆQ‰ˆØ€Hr:   c                 ó†   — t        | |z   |z   ||dz
  z  z
  dz
  |r|dz
  ndz   |«      dz   }|r|dz
  |z  | |z   k\  r|dz  }|S rð  )r{  )Ú	inputSizeÚ
kernelSizer  r	  rÆ   rÆ  r5  Ú
outputSizes           r8   Úpooling_output_shape_pad_lrr€  ‡  sŒ   € ô 	ØØñàñð ˜* q™.Ñ)ñ*ð ñ	ñ
 'ˆv˜Šz¨Añ/ð ó	
ð ñ		ð ñ Ø˜‰N˜fÑ$¨	°EÑ(9Ò9Ø˜!‰OˆJØÐr:   c           	      óê   ‡‡‡— t        j                  |dk7  d„ «       t        j                  ‰dk\  ˆfd„«       t        j                  ‰‰dz
  ‰z  dz   dz  k  ˆˆˆfd„«       t        | ‰‰‰|‰|«      S )Nr   c                   ó   — y)Nzstride should not be zeror4   r4   r:   r8   re   z&pooling_output_shape.<locals>.<lambda>£  rm   r:   c                  ó   •— d‰ › �S )Nz'pad must be non-negative, but got pad: r4   ©Úpads   €r8   re   z&pooling_output_shape.<locals>.<lambda>¤  s   ø€ Ð%LÈSÈEÐ#R€ r:   r3   rý   c                  ó   •— d‰› d‰› d‰ › �S )NzApad should be at most half of effective kernel size, but got pad=z, kernel_size=z and dilation=r4   )rÆ  r~  r…  s   €€€r8   re   z&pooling_output_shape.<locals>.<lambda>§  s'   ø€ ØOÐPSÈuð UØ%˜, n°X°Jð@ð r:   )rV   rg   r€  )r}  r~  r…  rÆ   rÆ  r5  s    `` ` r8   rh  rh  ¢  ss   ú€ Ü	‡L�L�˜1‘ÑAÔBÜ	‡L�L�˜‘ÓRÔSÜ	‡L�LØ�˜a‘ 8Ñ+¨aÑ/°AÑ5Ñ5õ	
ôô 'Ø�:˜s C¨°¸9óð r:   c           	      ó\  ‡ ‡‡‡‡‡‡‡‡‡	‡
‡‡‡‡— ‰ j                  «       }‰	Št        j                  ‰dkD  xr ‰dkD  ˆˆfd„«       t        j                  ‰dkD  xr ‰dkD  ˆˆfd„«       t        j                  ‰dkD  xr ‰dkD  ˆˆfd„«       ‰ j                  d«      dk7  xr ‰ j                  d«      dk7  }|t        j                  k(  r7t        j                  |dk(  xr |xr ‰ j                  d«      dk7  ˆ fd	„«       nWt        j                  |dk(  xr ‰ j                  d«      dk7  xr |xs |dk(  xr |xr ‰ j                  d«      dk7  ˆ fd
„«       t        j                  ‰dz  ‰k\  xr ‰dz  ‰k\  ˆˆˆˆfd„«       t        j                  ‰dk\  xr ‰dk\  ˆ
ˆˆ	ˆˆˆfd„«       y )Nr   c                  ó   •— d‰ › d‰› �S )Nz5kernel size should be greater than zero, but got kH: ú, kW: r4   )rl  rm  s   €€r8   re   z$pool2d_shape_check.<locals>.<lambda>Ç  s   ø€ ÐGÈÀtÈ6ÐRTÐQUÐV€ r:   c                  ó   •— d‰ › d‰› �S )Nz0stride should be greater than zero, but got dH: ú, dW: r4   )rn  ro  s   €€r8   re   z$pool2d_shape_check.<locals>.<lambda>Ë  s   ø€ ÐBÀ2À$ÀfÈRÈDÐQ€ r:   c                  ó   •— d‰ › d‰› �S )Nz9dilation should be greater than zero, but got dilationH: ú, dilationW: r4   )Ú	dilationHÚ	dilationWs   €€r8   re   z$pool2d_shape_check.<locals>.<lambda>Ï  s   ø€ ÐKÈIÈ;ÐVcÐdmÐcnÐo€ r:   r3   rý   r5  r0   c                  ó*   •— d‰ j                  «       › �S )Nz€Expected 4D (batch mode) tensor expected for input with channels_last layout with optional 0 dim batch size for input, but got: r¢   r  s   €r8   re   z$pool2d_shape_check.<locals>.<lambda>×  s   ø€ ð CØCHÇ:Á:Ã<À.ðR€ r:   c                  ó*   •— d‰ j                  «       › �S )NzYExpected 3D or 4D (batch mode) tensor with optional 0 dim batch size for input, but got: r¢   r  s   €r8   re   z$pool2d_shape_check.<locals>.<lambda>Þ  s   ø€ ÐoÐpu×pzÑpzÓp|Ðo}Ð~€ r:   c                  ó    •— d‰› d‰› d‰› d‰ › �S )NzKpad should be smaller than or equal to half of kernel size, but got padW = z	, padH = z, kW = z, kH = r4   )rl  rm  rp  rq  s   €€€€r8   re   z$pool2d_shape_check.<locals>.<lambda>ã  s&   ø€ ð Ø��y   g¨b¨T°¸¸ð>€ r:   c                  ó.   •— d‰› d‰ › d‰› d‰› d‰› d‰› d�S ©NzGiven input size: (r=   z). Calculated output size: (z). Output size is too smallr4   )rr  rs  r6  rx  r7  r8  s   €€€€€€r8   re   z$pool2d_shape_check.<locals>.<lambda>é  s:   ø€ Ð% k ]°!°K°=ÀÀ*Àð N$Ø$0 >°°<°.ÀÀ+Àð O#ð#€ r:   )r†   rV   rg   r£   rà  )rØ   rl  rm  rn  ro  rp  rq  rŽ  r�  r6  rr  rs  r7  r8  r¬   r¥   Ú
valid_dimsrx  s   ``````````````   @r8   ri  ri  ±  sƒ  ÿþ€ ð" �9‰9‹;€DØ€Lä	‡L�LØ
ˆQ‰Ò�2˜‘6ÜVôô 
‡L�LØ
ˆQ‰Ò�2˜‘6ÜQôô 
‡L�LØ�A‰Ò'˜) a™-Üoôð
 —‘˜A“ !Ñ#Ò:¨¯
©
°1«¸Ñ(:€Jàœ×+Ñ+Ò+Ü�‰Ø�A‰IÒ;˜*Ò;¨¯©°A«¸!Ñ);óRõ	
ô 	�‰Ø�Q‰YÒ<˜5Ÿ:™: a›=¨AÑ-Ò<°*ò AØ˜‘	Ò?˜jÒ?¨U¯Z©Z¸«]¸aÑ-?Û~ô	
ô 
‡L�LØ
ˆa‰�4‰Ò+˜B !™G t™Oö	>ôô 
‡L�LØ�qÑÒ.˜\¨QÑ.÷	#ð 	#õr:   r”  r‘  rl  rm  r’  rn  ro  ÚpTÚpHÚpWÚ	dilationTrŽ  r�  r•  r–  r—  r˜  r™  rš  r�  c           
      óJ  ‡ ‡‡‡‡‡‡‡‡‡	‡
‡‡‡‡‡‡‡‡‡‡‡— ‰ j                   }t        j                  ‰dkD  xr ‰dkD  xr ‰dkD  ˆˆˆfd„«       t        j                  ‰dkD  xr ‰dkD  xr ‰dkD  ˆˆˆfd„«       t        j                  ‰dkD  xr ‰dkD  xr ‰dkD  ˆˆˆfd„«       t        j                  |dv ˆˆ fd„«       t        |«      D ]:  Š|dk(  r‰dk(  rŒt        j                  ‰ j	                  ‰«      dkD  ˆˆˆ fd„«       Œ< |r/t        j                  ‰‰k\  xr ‰‰k\  xr ‰‰k\  ˆˆˆˆˆˆfd	„«       t        j                  ‰d
z  ‰k\  xr ‰d
z  ‰
k\  xr ‰d
z  ‰	k\  ˆˆˆˆ	ˆˆ
fd„«       t        j                  ‰dk\  xr ‰dk\  xr ‰dk\  ˆˆˆˆˆˆˆfd„«       y )Nr   c                  ó   •— d‰› d‰ › d‰› �S )Nz5kernel size should be greater than zero, but got kT: z, kH: r‰  r4   )rl  r‘  rm  s   €€€r8   re   z$pool3d_shape_check.<locals>.<lambda>  s#   ø€ ðØ�$�f˜R˜D  r dð,ð r:   c                  ó   •— d‰› d‰ › d‰› �S )Nz0stride should be greater than zero, but got dT: z, dH: r‹  r4   )rn  r’  ro  s   €€€r8   re   z$pool3d_shape_check.<locals>.<lambda>  s    ø€ Ø>¸r¸dÀ&ÈÈÈFÐSUÐRVÐWð r:   c                  ó   •— d‰› d‰ › d‰› �S )Nz9dilation should be greater than zero, but got dilationT: z, dilationH: r�  r4   )rŽ  r™  r�  s   €€€r8   re   z$pool3d_shape_check.<locals>.<lambda>  s$   ø€ ðØ#˜ M°)°¸MÈ)ÈðVð r:   r‹  c                  ó&   •— ‰ › d‰j                   › �S )Nz/: Expected 4D or 5D tensor for input, but got: r$  )r  rØ   s   €€r8   re   z$pool3d_shape_check.<locals>.<lambda>   s   ø€ �7�)ÐJÈ5Ï;É;È-ÐX€ r:   rZ  c                  óL   •— ‰ › d‰j                   › d‰j                  ‰«      › d�S )NzZ: Expected input's non-batch dimensions to have positive length, but input has a shape of z and non-batch dimension z has length zero!)r™   r£   )r  rÜ   rØ   s   €€€r8   re   z$pool3d_shape_check.<locals>.<lambda>)  s3   ø€ Ø�)ð -Ø-2¯[©[¨MØ+¨E¯J©J°q«M¨?Ð:KðMð r:   c                  ó.   •— d‰› d‰ › d‰› d‰› d‰› d‰› d�S )Nzinput image (T: rb  r>  z ) smaller than kernel size (kT: ú kH: ú kW: ry   r4   )r–  r•  r—  rl  r‘  rm  s   €€€€€€r8   re   z$pool3d_shape_check.<locals>.<lambda>3  s9   ø€ Ø" 5 '¨¨g¨Y°d¸6¸(ð C$Ø$& 4 u¨R¨D°°b°T¸ð<ð r:   rý   c                  ó,   •— d‰› d‰› d‰ › d‰› d‰› d‰› �S )NzHpad should be smaller than or equal to half of kernel size, but got kT: r¢  r¡  z padT: z padW: z padH: r4   )rl  r‘  rm  r—  r–  r˜  s   €€€€€€r8   re   z$pool3d_shape_check.<locals>.<lambda>;  s6   ø€ ðØ�$�e˜B˜4˜u R D¨°¨t°7¸2¸$¸gÀbÀTðKð r:   r3   c                  ó:   •— d‰› d‰› d‰ › d‰› d‰› d‰› d‰› d‰› d�S r”  r4   )r–  r•  r—  r”  r™  r˜  rš  s   €€€€€€€r8   re   z$pool3d_shape_check.<locals>.<lambda>C  sI   ø€ Ø! ' ¨!¨E¨7°!°G°9¸A¸f¸Xð F(Ø(/ y°°%°¸¸'¸À!ÀFÀ8ð L'ð(ð r:   )r¥   rV   rg   rÅ   r£   )rØ   r”  r‘  rl  rm  r’  rn  ro  r–  r—  r˜  r™  rŽ  r�  r•  r–  r—  r˜  r™  rš  r  r�  r¥   rÜ   s   `````````````````````  @r8   r�  r�  ï  s’  ÿÿý€ ð0 �:‰:€Dä	‡L�LØ
ˆQ‰Ò$�2˜‘6Ò$˜b 1™fõ	
ôô 
‡L�LØ
ˆQ‰Ò$�2˜‘6Ò$˜b 1™fõ	
ôô 
‡L�LØ�A‰Ò9˜) a™-Ò9¨I¸©Mõ	
ôô 
‡L�LØ�ˆÜXôô
 �4‹[ò 
ˆØ�1Š9˜˜ašàÜ�‰Ø�J‰J�q‹M˜AÑõõ	
ð	
ñ Ü�‰Ø�R‰KÒ:˜G r™MÒ:¨f¸©l÷ð ô	
ô 
‡L�LØ
ˆQ‰�"‰Ò6˜˜a™ 2™Ò6¨"¨q©&°B©,÷	
ð 	
ôô 
‡L�LØ�‰
Ò3�v ‘{Ò3 w°!¡|÷	
ñ 	
õr:   c                 ój  — | j                   }t        | |||||||	|
||||||||||||«       t        |||dz
  |«       t        |||dz
  |«       t        |||dz
  |«       t        |||dz
  |«       t        |||dz
  |«       t        |||dz
  |«       t        |||dz
  |«       t        |||dz
  |«       y )Nr5  r0   rý   r3   ©r¥   r�  rN  )rØ   r*  r´   r”  r‘  rl  rm  r’  rn  ro  r–  r—  r˜  r™  rŽ  r�  r•  r–  r—  r˜  r™  rš  r  r¥   s                           r8   Úmax_pool3d_backward_shape_checkr§  K  sæ   € ð2 �:‰:€DäØØØ
Ø
Ø
Ø
Ø
Ø
Ø
Ø
Ø
ØØØØØØØØØØô+ô0 �;  d¨Q¡h°Ô8Ü�;  d¨Q¡h°Ô6Ü�;  d¨Q¡h°Ô8Ü�;  d¨Q¡h°Ô7ä�7˜D $¨¡(¨GÔ4Ü�7˜D $¨¡(¨EÔ2Ü�7˜D $¨¡(¨GÔ4Ü�7˜D $¨¡(¨FÕ3r:   c                 óä   — | j                   }t        | ||||||||	|
|ddd|||||||d«       t        |||dz
  |«       t        |||dz
  |«       t        |||dz
  |«       t        |||dz
  |«       y )Nr3   Tr5  r0   rý   r¦  )rØ   r*  r”  r‘  rl  rm  r’  rn  ro  r–  r—  r˜  r•  r–  r—  r˜  r™  rš  r  r¥   s                       r8   r¢  r¢  ‰  s¡   € ð* �:‰:€DäØØØ
Ø
Ø
Ø
Ø
Ø
Ø
Ø
Ø
Ø	Ø	Ø	ØØØØØØØØô-ô2 �;  d¨Q¡h°Ô8Ü�;  d¨Q¡h°Ô6Ü�;  d¨Q¡h°Ô8Ü�;  d¨Q¡h°Õ7r:   c                 óú  — d„ } |d|«      \  }}t        j                  t        |«      dv d„ «       t        |«      dk(  r||}
}	n |d|«      \  }	}
 |d|«      \  }} |d|«      \  }}| j                  d	«      }| j                  d
«      }| j                  d«      }t	        j
                  | «      }|t         j                  k(  r)t        j                  | j                  «       dk(  d„ «       nR|t         j                  k(  r(t        j                  | j                  «       dv d„ «       nt        j                  dd„ «       t        ||||	||«      }t        ||||
||«      }t        | |||	|
||||||||||«       |||fS )Nc                 óˆ   ‡ — t        j                  t        |«      dv ˆ fd„«       |d   }t        |«      dk(  r|n|d   }||fS )NrS  c                  ó   •— d‰ › d�S )Nzmax_pool2d: rU  r4   rV  s   €r8   re   zEmax_pool2d_checks_and_compute_shape.<locals>.unpack.<locals>.<lambda>Ë  rW  r:   r   r3   rX  rY  s   `   r8   r\  z3max_pool2d_checks_and_compute_shape.<locals>.unpackÈ  r]  r:   rØ  r^  c                   ó   — y)NzOmax_pool2d: stride must either be omitted, a single int, or a tuple of two intsr4   r4   r:   r8   re   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>Õ  rm   r:   r   rÆ   rù  rÆ  re  rò  rœ   r5  c                   ó   — y)NzMnon-empty 4D (batch mode) tensor expected for input with channels_last layoutr4   r4   r:   r8   re   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>æ  rm   r:   rÁ  c                   ó   — y)Nz9non-empty 3D or 4D (batch mode) tensor expected for inputr4   r4   r:   r8   re   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>ë  rm   r:   Fc                   ó   — y)NzAUnsupported memory format. Supports only ChannelsLast, Contiguousr4   r4   r:   r8   re   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>ð  rm   r:   )rV   rg   rÃ   r£   rO   r   rà  r†   r°   rh  ri  )rØ   rØ  rÆ   rù  rÆ  r5  r\  rl  rm  rn  ro  rp  rq  rŽ  r�  r6  rr  rs  r¬   r7  r8  s                        r8   r3  r3  ¿  s�  € òñ �M ;Ó/�F€Bˆä	‡L�LÜˆF‹�yÐ Ùaôô ˆ6ƒ{�aÒØ�RˆB‰á˜ &Ó)‰ˆˆBá˜	 7Ó+�J€Dˆ$Ù! *¨hÓ7Ñ€IˆyØ—*‘*˜R“.€KØ—*‘*˜R“.€KØ—‘˜B“€Jä×/Ñ/°Ó6€MØœ×+Ñ+Ò+Ü�‰Ø�I‰I‹K˜1ÑÙcõ	
ð 
œ%×1Ñ1Ò	1Ü�‰Ø�I‰I‹K˜6Ð!ÙOõ	
ô
 	�‰ØÙWô	
ô
 (¨°R¸¸rÀ9ÈiÓX€LÜ& z°2°t¸RÀÈIÓV€KäØØ
Ø
Ø
Ø
ØØØØØØØØØØôð$ ˜ kÐ1Ð1r:   c                 ó|  ‡ ‡‡‡‡‡— t        ‰|||||«      \  }ŠŠt        j                  ‰j                  ‰ j                  k(  ˆ ˆfd„«       |Š‰j                  Šˆˆˆˆfd„}	 |	‰ «        |	|«       t        j                  ‰«      }
t        j                  ‰j                  ‰j                  ‰j                  |
¬«      S )Nc                  ó<   •— d‰j                   › d‰ j                   › �S )NzExpected dtype z  for `gradOutput` but got dtype r‘   r¼  s   €€r8   re   z7meta_max_pool2d_with_indices_backward.<locals>.<lambda>!  s    ø€ �/ $§*¡* Ð-MÈk×N_ÑN_ÐM`Ða€ r:   c                 ól   •— t        | ‰‰dz
  ‰«       t        | ‰‰dz
  ‰«       t        | ‰‰dz
  ‰«       y )Nr0   rý   r3   )rN  )r  rx  r¥   r7  r8  s    €€€€r8   Ú_check_dim_sizez>meta_max_pool2d_with_indices_backward.<locals>._check_dim_size'  s9   ø€ Ü�q˜$  q¡¨,Ô7Ü�q˜$  q¡¨,Ô7Ü�q˜$  q¡¨+Õ6r:   r¯  )
r3  rV   rg   r^   r¥   rO   r   rŒ   r™   rƒ   )r*  rš   rØ  rÆ   rù  rÆ  r5  r´   r6  r³  r¬   rx  r¥   r7  r8  s   ``         @@@@r8   Ú%meta_max_pool2d_with_indices_backwardr´    s«   ý€ ô  	,Øˆk˜6 7¨H°ió	ñ	ØØØô
 
‡L�LØ�
‰
�k×'Ñ'Ñ'Üaôð
 €LØ�9‰9€D÷7ñ
 �KÔ Ù�GÔä×/Ñ/°Ó5€MÜ�;‰;Ø�
‰
Ø�j‰jØ�{‰{Ø#ô	ð r:   c                 óš  — t        | |||||«      \  }}}| j                  «       dk(  r| j                  d«      nd}	t        j                  | «      }
| j                  «       dk(  r|||g}n|	|||g}t        j                  || j                  | j                  |
¬«      t        j                  |t
        j                  | j                  |
¬«      fS r1  )
r3  r†   r£   rO   r   rV   rŒ   r^   rƒ   r·   r4  s               r8   Úmeta_max_pool2d_with_indicesr¶  8  sÉ   € ô 	,Øˆ{˜F G¨X°yó	ñ	ØØØð
  %Ÿy™y›{¨aÒ/ˆU�Z‰Z˜Œ^°Q€FÜ×/Ñ/°Ó6€MØ‡y�yƒ{�aÒØ˜\¨;Ð7‰à˜ \°;Ð?ˆä�‰ØØ—+‘+Ø—<‘<Ø'ô		
ô 	�‰ØÜ—+‘+Ø—<‘<Ø'ô		
ðð r:   c                 ót  ‡ ‡‡‡
‡‡— t        j                  ‰ j                  dv ˆ fd„«       ‰ j                  }t        |dz
  |«      D ].  Š
t        j                  ‰ j	                  ‰
«      dkD  ˆ
ˆ fd„«       Œ0 t        j                  t        ‰«      dk(  d„ «       t        j                  t        |«      dk(  d„ «       ‰ j	                  d	«      }‰ j	                  d
«      Š‰ j	                  d«      Š|dk(  r‰ j	                  d«      }nd}t        j                  ‰ j                  ‰j                  k(  d„ «       t        j                  ‰j                  dk(  ˆfd„«       ‰j	                  d«      }‰j	                  d«      }‰j	                  d«      Š
t        j                  ||k\  d„ «       t        j                  ||k(  d„ «       t        j                  ‰
dk(  ˆ
fd„«       t        j                  |d   ‰d   z   dz
  ‰k  ˆˆfd„«       t        j                  |d   ‰d   z   dz
  ‰k  ˆˆfd„«       ‰ j                  «       dk(  r|||d   |d   g}	n||d   |d   g}	t        j                  |	‰ j                  ‰ j                  ¬«      t        j                  |	t         j                  ‰ j                  ¬«      fS )NrÁ  c                  ó"   •— d‰ j                   › �S )Nz:fractional_max_pool2d: Expected 3D or 4D tensor, but got: r7  r±   s   €r8   re   z,meta_fractional_max_pool2d.<locals>.<lambda>c  s   ø€ ÐLÈTÏYÉYÈKÐX€ r:   r0   r   c                  ó2   •— d‰j                  «       › d‰ › d�S )Nz_fractional_max_pool2d: Expected input to have non-zero  size for non-batch dimensions, but got r±  z emptyr¢   )rÔ   rš   s   €€r8   re   z,meta_fractional_max_pool2d.<locals>.<lambda>j  s'   ø€ ð 7Ø7;·y±y³{°mÐCSÐTUÐSVÐV\ð^€ r:   rý   c                   ó   — y)NzNfractional_max_pool2d: kernel_size musteither be a single int or tuple of Intsr4   r4   r:   r8   re   z,meta_fractional_max_pool2d.<locals>.<lambda>q  rm   r:   c                   ó   — y)NzOfractional_max_pool2d: output_size must either be a single int or tuple of Intsr4   r4   r:   r8   re   z,meta_fractional_max_pool2d.<locals>.<lambda>v  rm   r:   re  rò  rœ   r5  r3   c                   ó   — y)Nz6Expect _random_samples to have the same dtype as inputr4   r4   r:   r8   re   z,meta_fractional_max_pool2d.<locals>.<lambda>„  rm   r:   c                  ó"   •— d‰ j                   › �S )Nz1Expect _random samples to have 3 dimensions got, r7  )Úrandom_sampless   €r8   re   z,meta_fractional_max_pool2d.<locals>.<lambda>ˆ  s   ø€ ÐCÀN×DWÑDWÐCXÐY€ r:   c                   ó   — y)Nz=Expect _random_samples.size(0) no less then input batch size.r4   r4   r:   r8   re   z,meta_fractional_max_pool2d.<locals>.<lambda>�  rm   r:   c                   ó   — y)Nz<Expect _random_samples.size(1) equals to input channel size.r4   r4   r:   r8   re   z,meta_fractional_max_pool2d.<locals>.<lambda>”  rm   r:   c                  ó   •— d‰ › d�S )Nz/Expect _random_samples.size(2) equals to 2 got ú.r4   )rÔ   s   €r8   re   z,meta_fractional_max_pool2d.<locals>.<lambda>–  s   ø€ Ð#RÐSTÐRUÐUVÐ!W€ r:   c                  ó   •— d‰d   › d‰ › �S )Nz%fractional_max_pool2d: kernel height r   z' is too large relative to input height r4   )Úinput_heightrØ  s   €€r8   re   z,meta_fractional_max_pool2d.<locals>.<lambda>š  s   ø€ Ð7¸ÀA¹Ð7GÐGnÐo{Ðn|Ð}€ r:   c                  ó   •— d‰d   › d‰ › �S )Nz$fractional_max_pool2d: kernel width r3   z& is too large relative to input width r4   )Úinput_widthrØ  s   €€r8   re   z,meta_fractional_max_pool2d.<locals>.<lambda>ž  s   ø€ Ð6°{À1±~Ð6FÐFlÐmxÐlyÐz€ r:   ré  )rV   rg   r¥   rÅ   r£   rÃ   r^   r†   rŒ   rƒ   r·   )rš   rØ  rª  r¾  r¥   Úinput_channelsÚinput_batchr  Úcr£   rÔ   rÄ  rÆ  s   `` `      @@@r8   Úmeta_fractional_max_pool2drÊ  _  ss  ý€ ä	‡L�LØ�	‰	�VÐÛXôð �9‰9€Dä�4˜!‘8˜TÓ"ò 
ˆÜ�‰Ø�I‰I�a‹L˜1Ñô^õ	
ð
ô 
‡L�LÜˆKÓ˜AÑñ	2ôô
 
‡L�LÜˆKÓ˜AÑñ	2ôð —Y‘Y˜r“]€NØ—9‘9˜R“=€LØ—)‘)˜B“-€KØˆq‚yØ—i‘i “l‰àˆä	‡L�LØ�
‰
�n×*Ñ*Ñ*ÙHôô 
‡L�LØ×Ñ˜qÑ ÛYôð
 	×Ñ˜AÓ€AØ×Ñ˜AÓ€AØ×Ñ˜AÓ€AÜ	‡L�LØ	ˆ[ÑÙOôô 
‡L�LØ	ˆ^ÑÙNôô 
‡L�L��a‘ÓWÔXä	‡L�LØ�A‰˜ Q™Ñ'¨!Ñ+¨|Ñ;Ü}ôô 
‡L�LØ�A‰˜ Q™Ñ'¨!Ñ+¨{Ñ:Üzôð
 ‡x�xƒz�Q‚Ø˜^¨[¸©^¸[È¹^ÐL‰à ¨A¡°¸A±Ð?ˆô 	�‰ØØ—*‘*Ø—;‘;ô	
ô
 	�‰ØÜ—+‘+Ø—;‘;ô	
ðð r:   c                 óô  — t        j                  t        |«      dv d„ «       |d   }t        |«      dk(  r|n|d   }t        |«      dk(  r|n|d   }t        j                  | xs t        |«      dv d„ «       |s|n|d   }	|s|nt        |«      dk(  r|	n|d   }
|s|nt        |«      dk(  r|	n|d   }t        j                  t        |«      dv d„ «       |d   }t        |«      dk(  r|n|d   }t        |«      dk(  r|n|d   }t        j                  t        |«      dv d„ «       |d   }t        |«      dk(  r|n|d   }t        |«      dk(  r|n|d   }t        j                  | j                  d	v d
„ «       | j                  dk(  r| j	                  d«      nd}| j	                  d«      }| j	                  d«      }| j	                  d«      }| j	                  d«      }t        ||||	||«      }t        ||||
||«      }t        ||||||«      }t        | |||||	|
|||||||||||||d«       | j                  dk(  xr& t        j                  | «      t         j                  k(  }| j                  dk(  r||||f}n|||||f}| j                  |«      }| j                  |t         j                  ¬«      }|r@|j                  t         j                  ¬«      }|j                  t         j                  ¬«      }||fS )Nrƒ  c                   ó   — y©NzMmax_pool3d: kernel_size must either be a single int, or a tuple of three intsr4   r4   r:   r8   re   z.meta_max_pool3d_with_indices.<locals>.<lambda>À  rm   r:   r   r3   rý   c                   ó   — y©NzQmax_pool3d: stride must either be omitted, a single int, or a tuple of three intsr4   r4   r:   r8   re   z.meta_max_pool3d_with_indices.<locals>.<lambda>È  rm   r:   c                   ó   — y©NzImax_pool3d: padding must either be a single int, or a tuple of three intsr4   r4   r:   r8   re   z.meta_max_pool3d_with_indices.<locals>.<lambda>Ð  rm   r:   c                   ó   — y©NzJmax_pool3d: dilation must be either a single int, or a tuple of three intsr4   r4   r:   r8   re   z.meta_max_pool3d_with_indices.<locals>.<lambda>Ø  rm   r:   r‹  c                   ó   — yr�  r4   r4   r:   r8   re   z.meta_max_pool3d_with_indices.<locals>.<lambda>à  rm   r:   rZ  éûÿÿÿr2  re  rò  rœ   zmax_pool3d_with_indices()r5  r‘   r«   )rV   rg   rÃ   r¥   r£   rh  r�  rO   r   rû  r˜   r·   rY  )rØ   rØ  rÆ   rù  rÆ  r5  r‘  rl  rm  r’  rn  ro  r–  r—  r˜  r™  rŽ  r�  r  r”  r•  r–  r—  r˜  r™  rš  rà  r¨   rÍ   r´   s                                 r8   Úmeta_max_pool3d_with_indicesrÖ  ´  s  € ô 
‡L�LÜˆKÓ˜FÐ"Ù_ôð 
�Q‰€BÜ�;Ó 1Ò$‰¨+°a©.€BÜ�;Ó 1Ò$‰¨+°a©.€Bä	‡L�LØˆ
Ò+”c˜&“k VÐ+Ùcôñ ‰˜v a™y€BÙ‰¤c¨&£k°QÒ&6¡¸FÀ1¹I€BÙ‰¤c¨&£k°QÒ&6¡¸FÀ1¹I€Bä	‡L�LÜˆG‹˜ÐÙ[ôð 
�‰€BÜ�7‹|˜qÒ ‰ g¨a¡j€BÜ�7‹|˜qÒ ‰ g¨a¡j€Bä	‡L�LÜˆH‹˜ÐÙ\ôð ˜‘€IÜ  ›]¨aÒ/‘	°X¸a±[€IÜ  ›]¨aÒ/‘	°X¸a±[€Iä	‡L�LØ�
‰
�fÐÙKôð
  %Ÿz™z¨QšˆU�Z‰Z˜Œ^°A€FØ�j‰j˜‹n€GØ�J‰J�r‹N€EØ�j‰j˜‹n€GØ�Z‰Z˜‹^€Fä  ¨¨B°°I¸yÓI€EÜ" 7¨B°°B¸	À9ÓM€GÜ! &¨"¨b°"°iÀÓK€FäØØØ
Ø
Ø
Ø
Ø
Ø
Ø
Ø
Ø
ØØØØØØØØØØ#ô+ð4 	�
‰
�a‰ÒXœE×7Ñ7¸Ó>Ä%×BXÑBXÑXð ð ‡z�z�Q‚Ø˜e W¨fÐ5‰	à˜W e¨W°fÐ=ˆ	à
�/‰/˜)Ó
$€CØ�o‰o˜i¬u¯{©{ˆoÓ;€GáØ�f‰f¤5×#9Ñ#9ˆfÓ:ˆØ—*‘*¬5×+AÑ+A�*ÓBˆà�ˆ<Ðr:   c                 ó  — t        j                  t        |«      dv d„ «       |d   }t        |«      dk(  r|n|d   }	t        |«      dk(  r|n|d   }
t        j                  | xs t        |«      dv d„ «       |s|n|d   }|s|	nt        |«      dk(  r|n|d   }|s|
nt        |«      dk(  r|n|d   }t        j                  t        |«      dv d„ «       |d   }t        |«      dk(  r|n|d   }t        |«      dk(  r|n|d   }t        j                  t        |«      dv d„ «       |d   }t        |«      dk(  r|n|d   }t        |«      dk(  r|n|d   }t        j                  |j                  d	v d
„ «       |j	                  d«      }|j	                  d«      }|j	                  d«      }|j	                  d«      }| j	                  d«      }| j	                  d«      }| j	                  d«      }t        || ||||	|
|||||||||||||||d«       |j                  dk(  xr& t        j                  |«      t         j                  k(  }|j                  |j                  «      }|r |j                  t         j                  ¬«      }|S )Nrƒ  c                   ó   — yrÍ  r4   r4   r:   r8   re   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>&  rm   r:   r   r3   rý   c                   ó   — yrÏ  r4   r4   r:   r8   re   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>.  rm   r:   c                   ó   — yrÑ  r4   r4   r:   r8   re   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>6  rm   r:   c                   ó   — yrÓ  r4   r4   r:   r8   re   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>>  rm   r:   r‹  c                   ó   — yr�  r4   r4   r:   r8   re   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>F  rm   r:   r2  re  rò  rœ   z"max_pool3d_with_indices_backward()rZ  r«   )rV   rg   rÃ   r¥   r£   r§  rO   r   rû  r˜   r™   rY  )r*  rØ   rØ  rÆ   rù  rÆ  r5  r´   r‘  rl  rm  r’  rn  ro  r–  r—  r˜  r™  rŽ  r�  r”  r•  r–  r—  r˜  r™  rš  rà  r-  s                                r8   Ú%meta_max_pool3d_with_indices_backwardrÝ    s�  € ô 
‡L�LÜˆKÓ˜FÐ"Ù_ôð 
�Q‰€BÜ�;Ó 1Ò$‰¨+°a©.€BÜ�;Ó 1Ò$‰¨+°a©.€Bä	‡L�LØˆ
Ò+”c˜&“k VÐ+Ùcôñ ‰˜v a™y€BÙ‰¤c¨&£k°QÒ&6¡¸FÀ1¹I€BÙ‰¤c¨&£k°QÒ&6¡¸FÀ1¹I€Bä	‡L�LÜˆG‹˜ÐÙ[ôð 
�‰€BÜ�7‹|˜qÒ ‰ g¨a¡j€BÜ�7‹|˜qÒ ‰ g¨a¡j€Bä	‡L�LÜˆH‹˜ÐÙ\ôð ˜‘€IÜ  ›]¨aÒ/‘	°X¸a±[€IÜ  ›]¨aÒ/‘	°X¸a±[€Iä	‡L�LØ�
‰
�fÐÙKôð
 �j‰j˜‹n€GØ�J‰J�r‹N€EØ�j‰j˜‹n€GØ�Z‰Z˜‹^€Fà×Ñ˜RÓ €EØ×Ñ˜rÓ"€GØ×Ñ˜bÓ!€Fä#ØØØØØ
Ø
Ø
Ø
Ø
Ø
Ø
Ø
Ø
ØØØØØØØØØØ,ô/ð8 	�
‰
�a‰ÒXœE×7Ñ7¸Ó>Ä%×BXÑBXÑXð ð —‘ §¡Ó-€JáØ—]‘]´×1GÑ1G�]ÓHˆ
àÐr:   Úgridc                 óz  ‡ ‡‡— t        j                  ‰ j                  ‰j                  k(  ˆˆ fd„«       t        j                  ‰ j                  t         j                  k(  xr ‰j                  t         j                  k(  ˆˆ fd„«       t        j                  ‰ j
                  d   ‰j
                  d   k(  ˆˆ fd„«       t        j                  ‰j
                  d   ‰ j                  dz
  k(  ˆˆ fd„«       t        d‰ j                  «      D ],  Št        j                  ‰ j
                  ‰   dkD  ˆˆ fd„«       Œ. y )	Nc                  ó<   •— d‰j                   › d‰ j                   › �S )NzNgrid_sampler(): expected input and grid to be on same device, but input is on z and grid is on r7  ©rÞ  rØ   s   €€r8   re   z+check_grid_sampler_common.<locals>.<lambda>|  s'   ø€ ðØ—\‘\�NÐ"2°4·;±;°-ðAð r:   c                  ó<   •— d‰j                   › d‰ j                   › �S )NzTgrid_sampler(): expected input and grid to have torch.strided layout, but input has z and grid has )r‚   rá  s   €€r8   re   z+check_grid_sampler_common.<locals>.<lambda>ƒ  s&   ø€ ðØŸ™˜ n°T·[±[°MðCð r:   r   c                  ó<   •— d‰j                   › d‰ j                   › �S )NzZgrid_sampler(): expected grid and input to have same batch size, but got input with sizes ú and grid with sizes r$  rá  s   €€r8   re   z+check_grid_sampler_common.<locals>.<lambda>Š  s'   ø€ ð Ø %§¡˜}Ð,AÀ$Ç*Á*ÀðOð r:   rœ   rý   c                  óB   •— d‰j                   dz
  › d‰ j                  › �S )Nz+grid_sampler(): expected grid to have size rý   z, in last dimension, but got grid with sizes )r¥   r™   rá  s   €€r8   re   z+check_grid_sampler_common.<locals>.<lambda>‘  s,   ø€ Ø9¸%¿*¹*Àq¹.Ð9Ið J2Ø26·*±*°ð?ð r:   c                  ó*   •— d‰j                   › d‰ › d�S )NzYgrid_sampler(): expected input to have non-empty spatial dimensions, but input has sizes r±  r²  r$  rÅ  s   €€r8   re   z+check_grid_sampler_common.<locals>.<lambda>š  rÆ  r:   )rV   rg   rƒ   r‚   rØ  r™   r¥   rÅ   )rØ   rÞ  rÜ   s   ``@r8   Úcheck_grid_sampler_commonrç  y  sì   ú€ Ü	‡L�LØ�‰˜Ÿ™Ñ#ô	
ôô 
‡L�LØ�‰œŸ™Ñ%ÒF¨$¯+©+¼¿¹Ñ*Fô	
ôô 
‡L�LØ�‰�A‰˜$Ÿ*™* Q™-Ñ'ô	
ôô 
‡L�LØ�
‰
�2‰˜%Ÿ*™* q™.Ñ(ô	
ôô �1�e—j‘jÓ!ò 
ˆÜ�‰Ø�K‰K˜‰N˜QÑôõ	
ñ
r:   c                   ó   — e Zd ZdZdZdZy)ÚGridSamplerInterpolationr   r3   rý   N)r{   Ú
__module__Ú__qualname__ÚBILINEARÚNEARESTÚBICUBICr4   r:   r8   ré  ré  ¡  s   „ Ø€HØ€GØ�Gr:   ré  Úinterpolation_modec                 ó  ‡ ‡— t        j                  ‰ j                  dk(  xr ‰ j                  ‰j                  k(  ˆˆ fd„«       t        j                  ‰ j                  dk(  xr |t        j                  j
                  k(   d„ «       y )NrZ  c                  ó<   •— d‰j                   › d‰ j                   › �S )Nzdgrid_sampler(): expected 5D input and grid with same number of dimensions, but got input with sizes rä  r$  rá  s   €€r8   re   z'check_grid_sampler_3d.<locals>.<lambda>ª  s&   ø€ ð4Ø49·K±K°=Ø# D§J¡J <ð1ð r:   c                   ó   — y)Nz<grid_sampler(): bicubic interpolation only supports 4D inputr4   r4   r:   r8   re   z'check_grid_sampler_3d.<locals>.<lambda>µ  rm   r:   )rV   rg   r¥   ré  rî  rV  )rØ   rÞ  rï  s   `` r8   Úcheck_grid_sampler_3dró  §  sp   ù€ Ü	‡L�LØ�
‰
�a‰Ò3˜EŸJ™J¨$¯)©)Ñ3ô	
ôô 
‡L�Là�J‰J˜!‰Oò MØ"Ô&>×&FÑ&F×&LÑ&LÑLð	
ñ 	Oõr:   c                 ó²   — |d   }|r&t        j                  |t         j                  ¬«      }nd }t        j                  |t         j                  ¬«      }	||	fS ©Nr   r«   )rV   r×  r°   r¯   ©
r*  rØ   rÞ  rï  Úpadding_modeÚalign_cornersr  Úinput_requires_gradr-  Ú	grad_grids
             r8   Úgrid_sampler_2d_backward_metarû  ¹  sQ   € ð & a™.ÐÙÜ×%Ñ% e¼5×;RÑ;RÔS‰
àˆ
Ü× Ñ  ´U×5LÑ5LÔM€IØ˜	Ð"Ð"r:   c                 óö   — t        | |«       t        | ||«       | j                  d   }| j                  d   }|j                  d   }|j                  d   }|j                  d   }	| j                  |||||	f«      S )Nr   r3   rý   r0   )rç  ró  r™   r˜   )
rØ   rÞ  rï  r÷  rø  r¹  ÚCÚout_DÚout_HÚout_Ws
             r8   Úgrid_sampler_3dr  Ì  sv   € ô ˜e TÔ*Ü˜% Ð'9Ô:Ø�‰�A‰€AØ�‰�A‰€AØ�J‰J�q‰M€EØ�J‰J�q‰M€EØ�J‰J�q‰M€EØ�?‰?˜A˜q %¨°Ð6Ó7Ð7r:   rú  c                 óä   — t        ||«       t        |||«       |d   }|r&t        j                  |t        j                  ¬«      }nd }t        j
                  |t        j                  ¬«      }	||	fS rõ  )rç  ró  rV   r×  r?  r¯   rö  s
             r8   Úgrid_sampler_3d_backwardr  ß  sm   € ô ˜e TÔ*Ü˜% Ð'9Ô:Ø% a™.ÐÙÜ×%Ñ%Ø¤×!?Ñ!?ô
‰
ð ˆ
Ü× Ñ  ´U×5SÑ5SÔT€IØ�yÐ Ð r:   c                 óŽ   — |j                  d«      }|st        j                  |«      }||d<   t        j                  | g|¢­i |¤ŽS )Nr^   )r]   rO   Ú	get_dtyperV   rŒ   )r£   ræ  rQ   r(  r^   s        r8   Úfullr  ÷  sC   € à�J‰J�wÓ€EÙÜ—‘ 
Ó+ˆØ€Fˆ7�Oä�;‰;�tÐ-˜dÒ- fÑ-Ð-r:   c                 óN  — |t         j                  k(  rÛt        j                  |d u d„ «       t        j                  d|€| j                  n|||€| j
                  n||¬«      }| j                  r>|j                  | j                  «       | j                  «       | j                  «       «       n/|j                  | j                  «       | j                  «       d«       |j                  d«       |S t        j                  j                  | |||||¬«      }|j!                  d«       |S )Nc                   ó   — y)Nz9memory format option is only supported by strided tensorsr4   r4   r:   r8   re   zzeros_like.<locals>.<lambda>  rm   r:   r   r  TrÚ  )rV   Ú
sparse_coorg   rŒ   r^   rƒ   Ú	is_sparseÚsparse_resize_and_clear_r£   Ú
sparse_dimÚ	dense_dimr†   Ú_coalesced_r-   r¯   ÚdefaultÚfill_)rš   r^   r‚   rƒ   r„   r¬   rÇ  s          r8   r×  r×    sü   € ð ”×!Ñ!Ò!Ü�‰Ø˜TÐ!ÙOô	
ô
 �k‰kØØ % �$—*’*°5ØØ"( .�4—;’;°fØ!ô
ˆð �>Š>Ø×(Ñ(Ø—	‘	“˜TŸ_™_Ó.°·±Ó0@õð ×(Ñ(¨¯©«°d·h±h³jÀ!ÔDà�‰˜ÔØˆ
Ü
�/‰/×
!Ñ
!ØØØØØØ#ð "ó €Cð ‡I�Iˆa„LØ€Jr:   r�   c                ó²   — |€t        j                  «       }|€t        j                  «       }|€t         j                  }t        j                  | ||||¬«      S r  ©rV   rˆ   Úget_default_devicerØ  rŒ   ©r£   r^   r‚   rƒ   r„   r…   s         r8   Ú	meta_onesr  /  óT   € ð €}Ü×'Ñ'Ó)ˆØ€~Ü×)Ñ)Ó+ˆØ€~Ü—‘ˆÜ�;‰;Ø�E &°ÀJôð r:   c                ó²   — |€t        j                  «       }|€t        j                  «       }|€t         j                  }t        j                  | ||||¬«      S r  r  r  s         r8   Ú
meta_zerosr  E  r  r:   c                 ó   — t        | «      S rD   ©Ú_scatter_meta_output)rš   r\  r†   r“   s       r8   Úmeta_select_scatterr  [  ó   € ä Ó%Ð%r:   c                 ó   — t        | «      S rD   r  )rš   r\  r†   r}   r|   Ústeps         r8   Úmeta_slice_scatterr   `  r  r:   c                 ó˜   — ddl m}  || «      s(t        j                  | «      dk(  r| j	                  «       S t        j                  | «      S )Nr   rT  r3   )rV  rU  rV   rW  rï   rO   Úclone_preserve_strides)rš   rU  s     r8   r  r  e  s?   € ÝKñ
 ! Ô&¬5×+LÑ+LÈTÓ+RÐVWÒ+WØ�z‰z‹|ÐÜ×'Ñ'¨Ó-Ð-r:   Údim_post_exprÚwrap_scalarc           	      ó˜   — |dk  r|st        d|› d�«      ‚d}| }|dz
  }| |k  s| |kD  rt        d| › d|› d|› d�«      ‚| dk  r| |z  } | S )	Nr   zdim_post_expr=z <= 0 but wrap_scalar is Falser3   zdim z out of bounds (rx   ry   r  )r†   r#  r$  rô   rá  s        r8   r¸   r¸   q  sƒ   € Ø˜ÒÙÜ Ø   Ð/MÐNóð ð ˆØˆ.€CØ
˜!Ñ
€CØ
ˆS‚y�C˜#’IÜ˜t C 5Ð(8¸¸¸RÀ¸uÀAÐFÓGÐGØ
ˆQ‚wØˆ}ÑˆØ€Jr:   c                 óJ   — | j                  «       dk(  rdS | j                  |   S rê  rL  )r  r†   s     r8   Úensure_nonempty_sizer'  �  s!   € Ø—‘“˜1’ˆ1Ð. !§'¡'¨#¡,Ð.r:   c                 ó:  ‡ ‡‡‡— t        ‰ j                  «       d«      }t        ‰j                  «       d«      }t        j                  ||k(  d„ «       t	        |«      D ];  Š‰‰k7  sŒ	t        j                  t        ‰‰«      t        ‰ ‰«      k  ˆˆˆˆ fd„«       Œ= y )Nr3   c                   ó   — y)NzDIndex tensor must have the same number of dimensions as input tensorr4   r4   r:   r8   re   z$gather_shape_check.<locals>.<lambda>‹  rm   r:   c                  óN   •— d‰› d‰j                   › �d‰j                   › d‰ › �z   S )Nz!Size does not match at dimension z expected index ú to be no larger than self ú apart from dimension r$  )r†   rÜ   r“   rš   s   €€€€r8   re   z$gather_shape_check.<locals>.<lambda>‘  s7   ø€ Ð;¸A¸3Ð>NÈuÏ{É{ÈmÐ\Ø/°·
±
¨|Ð;QÐRUÐQVÐWñX€ r:   )rá  r†   rV   rg   rÅ   r'  )rš   r†   r“   Ú	self_dimsÚ
index_dimsrÜ   s   ```  @r8   Úgather_shape_checkr/  †  s„   û€ Ü�D—H‘H“J Ó"€IÜ�U—Y‘Y“[ !Ó$€JÜ	‡L�LØ�ZÑÙVôô �9Óò ˆØ�‹8Ü�L‰LÜ$ U¨AÓ.Ô2FÀtÈQÓ2OÑOöXõñr:   c                 óp  ‡— ddl m} t        || j                  «       «      } |‰j	                  «       dk(  «      }|s`t        j                  ‰j                  t
        j                  k(  xs ‰j                  t
        j                  k(  ˆfd„«       t        | |‰«       | j                  ‰j                  «      S )Nr   ró  c                  ó"   •— d‰ j                   › �S )Nz8gather(): Expected dtype int32/int64 for index, but got r‘   r’   s   €r8   re   zmeta_gather.<locals>.<lambda>Ÿ  s   ø€ ÐNÈuÏ{É{ÈmÐ\€ r:   )rV  rô  r¸   r†   r—   rV   rg   r^   r•   rã   r/  r˜   r™   )rš   r†   r“   Úsparse_gradrô  Úwrapped_dimÚis_index_emptys     `    r8   Úmeta_gatherr5  –  s‚   ø€ åDä   d§h¡h£jÓ1€KÙ# E§K¡K£M°QÑ$6Ó7€NÙÜ�‰Ø�K‰Kœ5Ÿ:™:Ñ%ÒA¨¯©¼¿	¹	Ñ)AÛ\ô	
ô 	˜4 ¨eÔ4Ø�>‰>˜%Ÿ+™+Ó&Ð&r:   c                 óº   — |r6| dk(  ry| dk(  ry| dk(  ry| dk(  ry| d	k(  ry
t        j                  dd„ «       y | dk(  ry| dk(  ryt        j                  dd„ «       y )Nr  Ú
REDUCE_ADDrÖ  ÚREDUCE_MULTIPLYrH  ÚREDUCE_MEANÚamaxÚREDUCE_MAXIMUMÚaminÚREDUCE_MINIMUMFc                   ó   — y)Nz=reduce argument must be either sum, prod, mean, amax or amin.r4   r4   r:   r8   re   z#get_operator_enum.<locals>.<lambda>´  rm   r:   ÚaddÚmultiplyc                   ó   — y)Nz/reduce argument must be either add or multiply.r4   r4   r:   r8   re   z#get_operator_enum.<locals>.<lambda>¼  rm   r:   r’  )Úreduce_Úuse_new_optionss     r8   Úget_operator_enumrD  ¦  s{   € ÙØ�eÒØØ˜ÒØ$Ø˜ÒØ Ø˜ÒØ#Ø˜ÒØ#Ü�‰ØÙSô	
ð 	à�eÒØØ˜
Ò"Ø$Ü�‰�UÑUÔVØr:   c                 óP  ‡ — ddl m}  ||j                  «       dk7  «      rSt        j                  |j
                  t        j                  k(  xs |j
                  t        j                  k(  ˆ fd„«       |�1t        j                  |j
                  |j
                  k(  ˆ fd„«       y y )Nr   )r  c                  ó   •— ‰ › d�S )Nz((): Expected dtype int32/int64 for indexr4   ©Úmethod_names   €r8   re   z,scatter_gather_dtype_check.<locals>.<lambda>Ç  s   ø€ �{�mÐ#KÐL€ r:   c                  ó   •— ‰ › d�S )Nz0(): Expected self.dtype to be equal to src.dtyper4   rG  s   €r8   re   z,scatter_gather_dtype_check.<locals>.<lambda>Í  s   ø€ �{�mÐ#SÐT€ r:   )rV  r  r—   rV   rg   r^   r•   rã   )rH  rš   r“   Úsrc_optr  s   `    r8   Úscatter_gather_dtype_checkrK  Á  sv   ø€ ÝCá�U—[‘[“] aÑ'Ô(Ü�‰Ø�K‰Kœ5Ÿ:™:Ñ%ÒA¨¯©¼¿	¹	Ñ)AÛLô	
ð
 ÐÜ�‰Ø�J‰J˜'Ÿ-™-Ñ'ÛTõ	
ð r:   c                 ó   — t        | d«      S r2   )rá  r�   s    r8   Úensure_nonempty_dimrM  Ñ  s   € Üˆs�A‹;Ðr:   c                 óÆ  ‡ ‡‡‡— ddl m}  |‰j                  «       dk(  «      ry t        j                  t        ‰ j                  «       «      t        ‰j                  «       «      k(  d„ «       t        ‰ j                  «       «      }t        |«      D ]>  }|‰k(  rŒ	t        ‰|«      }t        ‰ |«      }t        j                  ||k  ˆˆˆ fd„«       Œ@ ‰�Žt        j                  t        ‰ j                  «       «      t        ‰j                  «       «      k(  d„ «       t        |«      D ]7  }t        ‰|«      }t        ‰|«      }	t        j                  ||	k  ˆˆfd„«       Œ9 y y )Nr   ró  c                   ó   — y)NzCIndex tensor must have the same number of dimensions as self tensorr4   r4   r:   r8   re   z%scatter_shape_check.<locals>.<lambda>Ý  rm   r:   c                  óH   •— d‰j                   › d‰j                   › �d‰ › �z   S )NúExpected index r+  r,  r$  )r†   r“   rš   s   €€€r8   re   z%scatter_shape_check.<locals>.<lambda>ë  s,   ø€ �o e§k¡k ]Ð2MÈdÏjÉjÈ\ÐZØ& s eÐ,ñ-€ r:   c                   ó   — y)NzBIndex tensor must have the same number of dimensions as src tensorr4   r4   r:   r8   re   z%scatter_shape_check.<locals>.<lambda>ó  rm   r:   c                  ó<   •— d‰ j                   › d‰j                   › �S )NrQ  z to be no larger than src r$  )r“   rJ  s   €€r8   re   z%scatter_shape_check.<locals>.<lambda>ú  s!   ø€ ˜/¨%¯+©+¨Ð6PÐQX×Q^ÑQ^ÐP_Ð`€ r:   )	rV  rô  r—   rV   rg   rM  r†   rÅ   r'  )
rš   r†   r“   rJ  rô  r-  rÔ   Úindex_d_sizeÚself_d_sizeÚ
src_d_sizes
   ````      r8   Úscatter_shape_checkrW  Ö  s-  û€ ÝDá�e—k‘k“m qÑ(Ô)ØÜ	‡L�LÜ˜DŸH™H›JÓ'Ô+>¸u¿y¹y»{Ó+KÑKÙUôô
 $ D§H¡H£JÓ/€Iô �9Óò 	
ˆØ�Š8ØÜ+¨E°1Ó5ˆÜ*¨4°Ó3ˆÜ�‰Ø˜KÑ'õ-õ	
ð	
ð ÐÜ�‰Ü §¡£
Ó+Ô/BÀ7Ç;Á;Ã=Ó/QÑQÙXô	
ô �yÓ!ò 	ˆAÜ/°°qÓ9ˆLÜ-¨g°qÓ9ˆJÜ�L‰LØ 
Ñ*Ü`õñ	ð r:   c                 óŽ   — t        || j                  «       «      }t        d| ||«       t        | |||«       |�t	        ||«       y y )NÚscatter)r¸   r†   rK  rW  rD  )rš   r†   r“   r\  rB  rC  r3  s          r8   Úscatter_meta_implrZ  ÿ  sE   € Ü   d§h¡h£jÓ1€KÜ˜y¨$°°sÔ;Ü˜˜k¨5°#Ô6ØÐä˜' ?Õ3ð r:   c                 óV   — t        | |||d«       | j                  | j                  «      S ©Nr?  ©rZ  r˜   r™   ©rš   r†   r“   r\  s       r8   Úmeta_scatter_addr_    s%   € ä�d˜C ¨¨UÔ3Ø�>‰>˜$Ÿ*™*Ó%Ð%r:   c                 ó$   — t        | |||d«       | S r\  ©rZ  r^  s       r8   Úmeta_scatter_add_rb    s   € ä�d˜C ¨¨UÔ3Ø€Kr:   c                 ó’   — t        |t        j                  «      r|nd }t        | ||||«       | j	                  | j
                  «      S rD   )rp   rV   r   rZ  r˜   r™   ©rš   r†   r“   Úsrc_or_valuer�  r\  s         r8   Úmeta_scatterrf    s;   € ô % \´5·<±<Ô@‰,Àd€CÜ�d˜C ¨¨VÔ4Ø�>‰>˜$Ÿ*™*Ó%Ð%r:   c                 ó`   — t        |t        j                  «      r|nd }t        | ||||«       | S rD   )rp   rV   r   rZ  rd  s         r8   Úmeta_scatter_rh  #  s-   € ô % \´5·<±<Ô@‰,Àd€CÜ�d˜C ¨¨VÔ4Ø€Kr:   ÚqueryrÁ   rV  Ú	dropout_pÚ	is_causalÚreturn_debug_maskc           	      ó.  — | j                  d«      }| j                  d«      }| j                  d«      }	| j                  d«      }
|j                  d«      }t        j                  | «      }t        j                  |||	ft        j                  | j
                  ¬«      }|ra|
dkD  rdnd}t        j                  |	|z  «      }|dk  rd}n|dk  rd}t        j                  |||	|f| j                  | j
                  ¬«      }n,t        j                  d| j                  | j
                  ¬«      }t        j                  j                  rt        j                  j                  «       st        | «      d	k(  rMt        j                  d
t        j                  d¬«      }t        j                  d
t        j                  d¬«      }nLt        j                  dt        j                  d¬«      }t        j                  d
t        j                  d¬«      }||d d |	||||f	S )Nr   r3   rý   r0   ré  é@   é€   é   r   r4   r€   )r£   rV   r¯   rŒ   rZ   rƒ   rÕ  Úceilr^   rÖ  r×  ré   ró   rð   r•   r)  )ri  rÁ   rV  rj  rk  rl  r©  rÚ   Ú	num_headsÚmax_seqlen_batch_qÚhead_dimÚmax_seqlen_batch_kÚ	attentionÚ	logsumexpÚblocksize_cÚmax_seqlen_kÚ
debug_maskÚseedÚoffsets                      r8   Ú(meta__scaled_dot_product_flash_attentionr}  1  s­  € ð —‘˜A“€JØ—
‘
˜1“€IØŸ™ A›ÐØ�z‰z˜!‹}€HØŸ™ !›Ðä× Ñ  Ó'€IÜ—‘Ø	�YÐ 2Ð3Ü�k‰kØ�|‰|ô€Iñ Ø%¨š]‘c°ˆÜ—y‘yÐ!3°kÑ!AÓBˆØ Ò$Ø‰LØ 3Ò&ØˆLÜ—[‘[Ø˜Ð$6¸ÐEØ—+‘+Ø—<‘<ô
‰
ô —[‘[ ¨%¯+©+¸e¿l¹lÔKˆ
ô ‡}�}×ÒœUŸZ™Z×4Ñ4Ô6¼+ÀeÓ:LÐPUÒ:Uä�{‰{˜2¤U§Z¡Z¸Ô?ˆÜ—‘˜R¤u§z¡z¸&ÔA‰ä�{‰{˜A¤e§l¡l¸6ÔBˆÜ—‘˜R¤u§|¡|¸FÔCˆð 	ØØØØØØØØð
ð 
r:   Ú	q_descaleÚ	k_descaleÚ	v_descalec
           	      óœ   — | j                   t        j                  k(  r| j                  t        j                  «      } t        | ||||||	«      S rD   )r^   rV   r•  rY  r”  r}  )
ri  rÁ   rV  r~  r  r€  rj  rk  rl  r©  s
             r8   Ú2meta__scaled_dot_product_flash_attention_quantizedr‚  q  sJ   € ð ‡{�{”e×)Ñ)Ò)Ø—‘œŸ™Ó(ˆä3ØØØØØØØóð r:   Ú	res_shape.c                 óš  ‡ — t        ‰ j                  «      |k(  rt        j                  ‰ «      }|S t	        g d¢ˆ fd„d¬«      }|D �cg c]  }||   ‘Œ	 }}t        t        |«      «      D �cg c]  }|j                  |«      ‘Œ }}t        j                  |‰ j                  ‰ j                  ¬«      j                  |«      }|S c c}w c c}w )N)r   r3   rý   r0   c                 ó*   •— ‰j                  «       |    S rD   rë  )Úidxri  s    €r8   re   z,alloc_with_matching_layout.<locals>.<lambda>”  s   ø€ ¨%¯,©,«.¸Ñ*=€ r:   TrÀ   ré  )rf   r™   rV   r¯   rä   rÅ   rÃ   r“   rŒ   r^   rƒ   rÈ   )ri  rƒ  rÇ  Ú	dim_orderr†  Úpermuted_shaperÜ   Úfinal_permutes   `       r8   Úalloc_with_matching_layoutrŠ  Œ  s¸   ø€ ô ˆU�[‰[Ó˜YÒ&Ü×Ñ˜uÓ%ˆð €Jô ÚÓ=Àtô
ˆ	ð 5>Ö>¨S˜) C›.Ð>ˆÐ>Ü5:¼3¸y»>Ó5JÖK°˜Ÿ™¨Õ+ÐKˆÐKÜ�k‰kØ %§+¡+°e·l±lô
ç
‰'�-Ó
 ð 	ð €Jùò ?ùÚKs   ÁCÁ,CÚ	attn_biasÚcompute_log_sumexpc	           	      óæ  — | j                  d«      }	| j                  d«      }
| j                  d«      }|j                  d«      }|j                  d«      }|	|
||f}t        | |«      }t        j                  |	|
|dft        j                  | j
                  ¬«      }t        j                  dt        j                  d¬«      }t        j                  dt        j                  d¬«      }||d d ||||d f	S )Nr   r3   rý   rœ   ré  r4   r€   )r£   rŠ  rV   rŒ   rZ   rƒ   r•   )ri  rÁ   rV  r‹  rŒ  rj  rk  rl  r©  r  rZ  ÚS_QÚS_KVÚD_Vrƒ  rÇ  Ú
logsum_expr{  r|  s                      r8   Ú(meta__scaled_dot_product_cudnn_attentionr’  Ÿ  sÛ   € ð 	�
‰
�1‹€AØ�
‰
�1‹€AØ
�*‰*�Q‹-€CØ�8‰8�A‹;€DØ
�*‰*�R‹.€Cà�A�s˜CÐ €IÜ
$ U¨IÓ
6€Cä—‘Ø	
ˆAˆs�AˆÜ�k‰kØ�|‰|ô€Jô �;‰;�r¤§¡°FÔ;€DÜ�[‰[˜¤5§:¡:°fÔ=€Fð 	ØØØØØØØØð
ð 
r:   c           	      ó€  — d\  }}	}
| j                  «       dk(  r| j                  «       \  }}	}
}n5| j                  «       dk(  r| j                  «       \  }	}
}d}nt        d«      ‚|j                  d«      }|j                  d«      }t        | j                  «      }||d<   t        | t        |«      «      }t        j                  ||	|
ft        j                  | j                  ¬«      }t        j                  d	t        j                  d
¬«      }t        j                  d	t        j                  d
¬«      }||d d |
|||d f	S )N)r   r   r   r5  r0   r3   zquery must be 3D or 4Drò  rœ   ré  r4   r€   )r†   r£   rX  rÄ   r™   rŠ  rf   rV   rŒ   rZ   rƒ   r•   )ri  rÁ   rV  r‹  rj  rk  rl  r©  r  ÚH_QrŽ  rR   r�  r�  r¨   rÇ  r‘  r{  r|  s                      r8   Ú5meta__scaled_dot_product_fused_attention_overrideabler•  Ë  s  € ð �K€A€sˆCØ‡y�yƒ{�aÒØŸ™›‰ˆˆ3�‘QØ	�‰‹˜Ò	Ø—j‘j“l‰ˆˆS�!Ø‰äÐ3Ó4Ð4Ø�8‰8�B‹<€DØ
�*‰*�R‹.€Cô �U—[‘[Ó!€IØ€Iˆb�MÜ
$ U¬E°)Ó,<Ó
=€Cä—‘Ø	
ˆC�ˆÜ�k‰kØ�|‰|ô€Jô �;‰;�r¤§¡°FÔ;€DÜ�[‰[˜¤5§:¡:°fÔ=€Fð 	ØØØØØØØØð
ð 
r:   )r©  r³  Úgrad_input_maskrw  Ú	cum_seq_qÚ	cum_seq_kÚmax_qÚmax_kÚphilox_seedÚphilox_offsetc                ó¾   — t        j                  |«      }t        j                  |«      }t        j                  |«      }|�t        j                  |«      nd }||||fS rD   rJ  )r³  ri  rÁ   rV  r‹  r–  rÍ   rw  r—  r˜  r™  rš  rj  rk  r›  rœ  r©  Úgrad_qÚgrad_krO  Úgrad_attn_biass                        r8   Ú>meta__scaled_dot_product_fused_attention_overrideable_backwardr¡  ÿ  sZ   € ô* ×Ñ˜eÓ$€FÜ×Ñ˜cÓ"€FÜ×Ñ˜eÓ$€FØ4=Ð4I”U×%Ñ% iÔ0Èt€NØ�6˜6 >Ð1Ð1r:   c                 óŠ   — t        j                  |«      }t        j                  |«      }t        j                  |«      }|||fS rD   rJ  )r³  ri  rÁ   rV  rÍ   rw  r—  r˜  r™  rš  rj  rk  r›  rœ  r©  rž  rŸ  rO  s                     r8   Ú'meta__scaled_dot_product_flash_backwardr£    sA   € ô, ×Ñ˜eÓ$€FÜ×Ñ˜cÓ"€FÜ×Ñ˜eÓ$€FØ�6˜6Ð!Ð!r:   Ú	attn_maskc                 ó   — | j                  d«      }| j                  d«      }| j                  d«      }	t        j                  | «      }
t        j                  ||	|ft        j                  | j
                  ¬«      j                  dd«      }|
|fS )Nr   r3   rý   ré  )r£   rV   r¯   rŒ   rZ   rƒ   r0  )ri  rÁ   rV  rj  rk  r¤  r©  rÚ   rr  rs  rv  rw  s               r8   Ú0meta__scaled_dot_product_flash_attention_for_cpur¦  7  sŠ   € ð —‘˜A“€JØ—
‘
˜1“€IØŸ™ A›Ðä× Ñ  Ó'€IÜ—‘àØØð	
ô
 �k‰kØ�|‰|ô÷ �i��1ƒoð ð 	Øðð r:   c
                 ón  — t        j                  |j                  «       d|j                  |j                  ¬«      }
t        j                  |j                  «       d|j                  |j                  ¬«      }t        j                  |j                  «       d|j                  |j                  ¬«      }|
||fS )N©r   rý   r3   r0   ré  )rV   Úempty_permutedr£   r^   rƒ   )r³  ri  rÁ   rV  rÍ   rw  rj  rk  r¤  r©  rž  rŸ  rO  s                r8   Ú9meta__scaled_dot_product_flash_attention_for_cpu_backwardrª  Y  s•   € ô& ×!Ñ!Ø�
‰
‹ØØ�k‰kØ�|‰|ô	€Fô ×!Ñ!Ø�‰‹
ØØ�i‰iØ�z‰zô	€Fô ×!Ñ!Ø�
‰
‹ØØ�k‰kØ�|‰|ô	€Fð �6˜6Ð!Ð!r:   Údropout_maskÚ
enable_gqac	                 óÄ   ‡ ‡‡‡‡‡‡‡— d„ }	 |	‰ «      \  ŠŠ |	|«      \  }
} |	|«      \  }}‰j                   \  ŠŠŠ}|j                   \  }}ŠŠˆˆˆˆˆˆ ˆˆfd„} |«       S )Nc                 ól  — | j                  «       dk(  r| j                  d«      dfS | j                  «       dkD  rxd}t        | j                  «       dz
  «      D ]  }|| j                  |   z  }Œ | j	                  || j                  d«      | j                  d«      | j                  d«      «      dfS | d	fS )
Nr0   r   Tr5  r3   re  rò  rœ   F)r†   r¿  rÅ   r™   Úviewr£   )r=   rÚ   rÜ   s      r8   Ú	ensure_4dzBmeta__scaled_dot_product_attention_math_for_mps.<locals>.ensure_4dŽ  sŸ   € Ø�5‰5‹7�aŠ<Ø—;‘;˜q“> 4Ð'Ð'Ø�U‰U‹W�qŠ[ØˆJÜ˜1Ÿ5™5›7 Q™;Ó'ò )�Ø˜aŸg™g a™jÑ(‘
ð)à—6‘6˜* a§f¡f¨R£j°!·&±&¸³*¸a¿f¹fÀR»jÓIÈ4ÐOÐOà�e�8ˆOr:   c                  óÞ  •— ‰j                  ‰‰‰‰f«      } ‰j                  ‰‰‰‰f«      }‰
r½‰	j                  «       dk(  r&| j                  d«      } |j                  d«      }| |fS t        ‰	j                  d d «      t        | j                  dd «      z   }t        ‰	j                  d d «      t        |j                  dd «      z   }| j                  |«      } |j                  |«      }| |fS )Nr0   r   re  r3   r5  )r˜   r†   ÚsqueezerÄ   r™   r¯  )rÍ   Úattnr¨   Ú
attn_shaperÚ   Úmax_seq_lengthÚnum_headÚq_Úq_sizeri  Ú
unsqueezedÚvalue_head_sizes       €€€€€€€€r8   Úsdpa_general_mpszImeta__scaled_dot_product_attention_math_for_mps.<locals>.sdpa_general_mps   sâ   ø€ Ø�l‰l˜J¨°&¸/ÐJÓKˆØ�|‰|˜Z¨°6¸>ÐJÓKˆÙØ�y‰y‹{˜aÒØ—k‘k !“n�Ø—|‘| A“�ð �DˆyÐô	 ! §¡¨S¨bÐ!1Ó2´T¸#¿)¹)ÀAÀa¸.Ó5IÑI�	Ü! %§+¡+¨c¨rÐ"2Ó3´d¸4¿:¹:ÀaÈ¸?Ó6KÑK�
Ø—h‘h˜yÓ)�Ø—y‘y Ó,�Ø�DˆyÐr:   r$  )ri  rÁ   rV  r¤  rj  rk  r«  r©  r¬  r°  Úk_rR   Úv_r»  rÚ   rµ  r¶  r·  r¸  r¹  rº  s   `             @@@@@@@r8   Ú/meta__scaled_dot_product_attention_math_for_mpsr¾  ‚  sl   ÿ€ ò	ñ ˜uÓ%�N€Bˆ
Ù�c‹N�E€BˆÙ�eÓ�E€Bˆà&(§h¡hÑ#€J�˜& !Ø,.¯H©HÑ)€A€qˆ.˜/÷ó ñ  ÓÐr:   c                 ó@  — | j                  dd«      } |j                  dd«      }|j                  dd«      }| j                  d«      }| j                  d«      }	| j                  d«      }
|j                  d«      }t        j                  ||	|
|| j                  | j
                  ¬«      }t        j                  j                  r&t        j                  j                  «       r	 |r|	nd}n|rt        j                  |	dz  «      dz  nd}t        j                  ||
|ft        j                  | j
                  ¬«      }|j                  dd«      }t        j                  dt        j                  d	¬«      }t        j                  dt        j                  d	¬«      }||||fS )
Nr3   rý   r   rò  rœ   ré  rU  r4   r€   )r0  r£   rV   rŒ   r^   rƒ   rÖ  r×  ré   ró   rÕ  rq  rZ   r•   )ri  rÁ   rV  r‹  rŒ  rj  rk  r©  r  rÒ  rr  ÚKvrÇ  Úlogsumexp_dimr‘  r{  r|  s                    r8   Ú,meta__scaled_dot_product_efficient_attentionrÂ  ³  sD  € ð �O‰O˜A˜qÓ!€EØ
�-‰-˜˜1Ó
€CØ�O‰O˜A˜qÓ!€Eà�
‰
�1‹€AØ�
‰
�1‹€AØ—
‘
˜2“€IØ	�‰�B‹€Bä
�+‰+�a˜˜I r°·±ÀUÇ\Á\Ô
R€Cä‡}�}×ÒœUŸZ™Z×4Ñ4Ô6ð	ñ 0™°Q‰á2DœŸ	™	 ! b¡&Ó)¨BÒ.È!ˆä—‘Ø	
ˆI�}Ð%Ü�k‰kØ�|‰|ô€Jð �-‰-˜˜1Ó
€Cô �;‰;�r¤§¡°FÔ;€DÜ�[‰[˜¤5§:¡:°fÔ=€Fà�
˜D &Ð(Ð(r:   c                 ó  — |j                  d«      }|j                  d«      }|j                  d«      }|j                  d«      }|j                  d«      }|j                  d«      }t        j                  ||||fd|j                  |j                  ¬«      }t        j                  ||||fd|j                  |j                  ¬«      }t        j                  ||||fd|j                  |j                  ¬«      }d }|�~|
d   ry|j                  d«      }|dz  dk(  r|n
|dz   |dz  z
  }t        |j                  «       «      }||d<   t        j                  ||j                  |j                  ¬«      }|d	d |…f   }||||fS )
Nr   r3   rý   r0   r¨  ré  rœ   rT  .)r£   rV   r©  r^   rƒ   rÄ   rŒ   )r³  ri  rÁ   rV  r‹  rÍ   rw  r›  rœ  rj  r–  rk  r©  rÚ   rr  r™  rt  Ú
head_dim_vrš  rž  rŸ  rO  Ú	grad_biasÚlastDimÚlastDimAlignedÚ	new_sizess                             r8   Ú+meta__scaled_dot_product_efficient_backwardrÉ  à  s{  € ð( —‘˜A“€JØ—
‘
˜1“€IØ�J‰J�q‹M€EØ�z‰z˜!‹}€HØ—‘˜A“€Jà�H‰H�Q‹K€Eä×!Ñ!Ø	�Y  xÐ0ØØ�k‰kØ�|‰|ô	€Fô ×!Ñ!Ø	�Y  xÐ0ØØ�i‰iØ�z‰zô	€Fô ×!Ñ!Ø	�Y  zÐ2ØØ�k‰kØ�|‰|ô	€Fð €IØÐ °Ò!3Ø—.‘. Ó$ˆØ$+¨b¡L°AÒ$5™¸7ÀR¹<È'ÐTVÉ,Ñ;VˆÜ˜Ÿ™Ó)Ó*ˆ	Ø&ˆ	�"‰Ü—K‘KØ˜YŸ_™_°Y×5EÑ5Eô
ˆ	ð ˜c 8 G 8˜mÑ,ˆ	à�6˜6 9Ð,Ð,r:   c                 óŠ   — t        j                  |«      }t        j                  |«      }t        j                  |«      }|||fS rD   rJ  )r³  ri  rÁ   rV  rÍ   rw  r›  rœ  r‹  r—  r˜  r™  rš  rj  rk  r©  rž  rŸ  rO  s                      r8   Ú'meta__scaled_dot_product_cudnn_backwardrË    sA   € ô. ×Ñ˜eÓ$€FÜ×Ñ˜cÓ"€FÜ×Ñ˜eÓ$€FØ�6˜6Ð!Ð!r:   Úwindow_size_leftÚwindow_size_rightÚ	seqused_kÚalibi_slopesÚblock_tabler(  c                 óÚ  — |€| j                  d«      n|j                  «       dz
  }|€| j                  d«      n|}|€|j                  d«      n|}| j                  d«      }| j                  d«      }t        j                  | «      }|€4t        j                  |||ft        j
                  | j                  ¬«      }nC| j                  d«      }t        j                  ||ft        j
                  | j                  ¬«      }|	ra|dkD  rdnd}t        j                  ||z  «      }|dk  rd}n|dk  rd}t        j                  ||||f| j                  | j                  ¬«      }n,t        j                  d| j                  | j                  ¬«      }d	\  }}t        j                  j                  rkt        j                  j                  «       rMt        j                  d
t        j                  d¬«      }t        j                  d
t        j                  d¬«      }nLt        j                  dt        j                  d¬«      }t        j                  d
t        j                  d¬«      }|||||fS )Nr   r3   rò  rœ   ré  rn  ro  rp  ©NNr4   r€   rý   )r£   r—   rV   r¯   rŒ   rZ   rƒ   rÕ  rq  r^   rÖ  r×  ré   ró   r•   r)  )ri  rÁ   rV  r—  r˜  r™  rš  rj  rk  rl  r©  rÌ  rÍ  rÎ  rÏ  rÐ  r(  rÚ   rs  ru  rr  rt  rv  rw  Útotal_qrx  ry  rz  r{  r|  s                                 r8   Úmeta__flash_attention_forwardrÔ  9  sõ  € ð8 #,Ð"3�—‘˜A”¸¿¹Ó9JÈQÑ9N€JØ*3Ð*;˜Ÿ™ AœÀÐØ(1Ð(9˜Ÿ™ !œ¸uÐØ—
‘
˜2“€IØ�z‰z˜"‹~€Hô × Ñ  Ó'€IØÐÜ—K‘KØ˜Ð$6Ð7Ü—+‘+Ø—<‘<ô
‰	ð —*‘*˜Q“-ˆÜ—K‘KØ˜Ð ¬¯©¸E¿L¹Lô
ˆ	ñ Ø%¨š]‘c°ˆÜ—y‘yÐ!3°kÑ!AÓBˆØ Ò$Ø‰LØ 3Ò&ØˆLÜ—[‘[Ø˜Ð$6¸ÐEØ—+‘+Ø—<‘<ô
‰
ô —[‘[ ¨%¯+©+¸e¿l¹lÔKˆ
ð �L€Dˆ&Ü‡}�}×ÒœUŸZ™Z×4Ñ4Ô6ä�{‰{˜2¤U§Z¡Z¸Ô?ˆÜ—‘˜R¤u§z¡z¸&ÔA‰ä�{‰{˜A¤e§l¡l¸6ÔBˆÜ—‘˜R¤u§|¡|¸FÔCˆàØØØØðð r:   c                 óF   — t        |||||||||	|
||||||«      \  }}}}}|S rD   )rÔ  )rÍ   ri  rÁ   rV  r—  r˜  r™  rš  rj  rk  rl  r©  rÌ  rÍ  rÎ  rÏ  rÐ  r(  rR   rw  s                       r8   Ú0meta__flash_attention_forward_no_dropout_inplacerÖ  ‹  sQ   € ô* :ØØØØØØØØØØØØØØØØó!Ñ€A€y�!�Q˜ð$ Ðr:   c                 ó¬   — | j                   t        j                  k(  r| j                  t        j                  «      } t        | |||||||||	|||||«      S rD   )r^   rV   r•  rY  r”  rÔ  )ri  rÁ   rV  r—  r˜  r™  rš  rj  rk  rl  r~  r  r€  r©  rÌ  rÍ  rÎ  rÏ  s                     r8   Ú'meta__flash_attention_forward_quantizedrØ  µ  sb   € ð* ‡{�{”e×)Ñ)Ò)Ø—‘œŸ™Ó(ˆä(ØØØØØØØØØØØØØØØóð r:   c                 óŠ   — t        j                  |«      }t        j                  |«      }t        j                  |«      }|||fS rD   rJ  )r³  ri  rÁ   rV  rÍ   rw  r—  r˜  r™  rš  rj  rk  r›  rœ  r©  rÌ  rÍ  Ú
grad_queryÚgrad_keyÚ
grad_values                       r8   Úmeta__flash_attention_backwardrÝ  à  sA   € ô0 ×!Ñ! %Ó(€JÜ×Ñ Ó$€HÜ×!Ñ! %Ó(€Jà�x Ð+Ð+r:   Úcu_seqlens_qÚcu_seqlens_kÚmax_seqlen_qry  Úcustom_mask_typeÚcausal_diagonalÚseqlen_kÚwindow_sizec                 ó¸  — | j                  d«      }| j                  d«      }|j                  d«      }| j                  d«      }|j                  d«      }t        j                  ||||| j                  | j                  ¬«      }|�|j                  d«      dz
  n|}|}|�|€t        d«      ‚|}|�|n|}|
rt        j                  |dz  «      dz  nd}t        j                  |||ft        j                  | j                  ¬«      }t        j                  dt        j                  d	¬«      }t        j                  dt        j                  d	¬«      }||||||fS )
Nr   r3   rò  rœ   ré  z;max_seqlen_q must not be None when cu_seqlens_q is providedrU  r4   r€   )
r£   rV   rŒ   r^   rƒ   rŠ   rÕ  rq  rZ   r•   )ri  rÁ   rV  rm  rÞ  rß  rà  ry  rj  rá  rŒ  r©  râ  rã  rä  r  rÒ  r¹  rr  rÀ  rÇ  Úlogsumexp_batch_dimÚactual_max_seqlen_qÚactual_max_seqlen_krÁ  r‘  r{  r|  s                               r8   Ú!meta__efficient_attention_forwardré  ÿ  sE  € ð, 	�
‰
�1‹€AØ�
‰
�1‹€AØ�‰�‹€AØ—
‘
˜2“€IØ	�‰�B‹€Bä
�+‰+�a˜˜I r°·±ÀUÇ\Á\Ô
R€Cà7CÐ7O˜,×+Ñ+¨AÓ.°Ò2ÐVWÐØÐØÐØÐÜ ØMóð ð +ÐØ*6Ð*B™,ÈÐá4FŒ�	‰	Ð%¨Ñ*Ó+¨bÒ0ÈAð ô —‘Ø	˜i¨Ð7Ü�k‰kØ�|‰|ô€Jô �;‰;�r¤§¡°FÔ;€DÜ�[‰[˜¤5§:¡:°fÔ=€Fà�
˜D &Ð*=Ð?RÐRÐRr:   Úbias_requires_gradÚnum_splits_keyÚshared_storage_dqdkdvc                 óÀ  — |rût        j                  |j                  d   |j                  d   k(  d„ «       t        j                  |j                  d   |j                  d   k(  d„ «       t        j                  g |j                  dd ¢d‘|j                  d   ‘|j                  d   ‘­|j                  |j
                  ¬«      }|j                  d	d«      }|j                  d	d«      }|j                  d	d
«      }n?t        j                  |«      }t        j                  |«      }t        j                  |«      }|�z|j                  d«      }|dz  dk(  r|n
|dz   |dz  z
  }t        |j                  «       «      }||d<   t        j                  ||j                  |j
                  ¬«      }|dd |…f   }n!t        j                  d|j
                  ¬«      }||||fS )Nr3   c                   ó   — y)Nz,seqlen must match for `shared_storage_dqdkdvr4   r4   r:   r8   re   z4meta__efficient_attention_backward.<locals>.<lambda>R  rm   r:   r0   c                   ó   — y)Nz3embedding dim must match for `shared_storage_dqdkdvr4   r4   r:   r8   re   z4meta__efficient_attention_backward.<locals>.<lambda>V  rm   r:   r   rò  rœ   ré  re  rý   rT  .r4   r7  )
rV   rg   r™   rŒ   r^   rƒ   r   r¯   r£   rÄ   )r³  ri  rÁ   rV  rm  rÞ  rß  rà  ry  rw  rj  r›  rœ  rá  rê  r©  rë  rì  ÚchunkrÚ  rÛ  rÜ  rÆ  rÇ  rÈ  rÅ  s                             r8   Ú"meta__efficient_attention_backwardrñ  6  s¤  € ñ2 Ü�‰Ø�K‰K˜‰N˜cŸi™i¨™lÑ*ÙBô	
ô 	�‰Ø�K‰K˜‰N˜cŸi™i¨™lÑ*ÙIô	
ô —‘ØEˆe�k‰k˜!˜BÐÐE ÐE E§K¡K°¡OÐE°U·[±[À±_ÑEØ—+‘+Ø—<‘<ô
ˆð
 —\‘\ " aÓ(ˆ
Ø—<‘<  AÓ&ˆØ—\‘\ " aÓ(‰
ä×%Ñ% eÓ,ˆ
Ü×#Ñ# CÓ(ˆÜ×%Ñ% eÓ,ˆ
àÐØ—)‘)˜B“-ˆØ$+¨b¡L°AÒ$5™¸7ÀR¹<È'ÐTVÉ,Ñ;VˆÜ˜Ÿ™›Ó%ˆ	Ø&ˆ	�"‰Ü—K‘K 	°·±ÀDÇKÁKÔPˆ	Ø˜c 8 G 8˜mÑ,‰	ä—K‘K ¨5¯<©<Ô8ˆ	à�x ¨YÐ6Ð6r:   Úscale_aÚscale_bÚscale_resultÚuse_fast_accumc                 ó2  ‡ ‡‡‡‡‡‡‡‡— d„ }t        j                  ‰ j                  «       dk(  xr ‰j                  «       dk(  ˆˆ fd„«       t        j                   |‰ j                  «      xr  |‰j                  «      ˆˆ fd„«       t	        ‰ «      dk(  st	        ‰ «      dk(  st	        ‰ «      dk(  �r’d„ }	d	„ }
d
„ }t	        ‰ «      dk7  rãt        j                   |	‰ j                  «       «      xs  |‰ «      ˆ fd„«       t        j                   |
‰j                  «       «      xs  |‰«      ˆfd„«       t        j                  ‰ j                  d«      dz  dk(  ˆ fd„«       t        j                  ‰j                  d«      dz  dk(  xr ‰j                  d«      dz  dk(  ˆfd„«       ‰ j                  \  ŠŠ‰j                  d«      Š‰j                  t         j                  k(  xr ‰j                  t         j                  k(  xs< ‰j                  t         j                  k(  xr ‰j                  t         j                  k(  }‰j                  «       dk(  rf‰j                  «       dk(  rSt        j                  ‰j                  t         j                  k(  xr ‰j                  t         j                  k(  d„ «       �n…|�r‰j                  t         j                  k(  rd}‰dz  Šn$d}‰ j                  t         j                  k(  r‰dz  Šd}t        ‰|«      }t        |d«      dz  }|t        ‰|«      z  |z  Š|t        ‰|«      z  |z  Š‰j                  «       ‰k(  r_‰j                  «       ‰k(  rLt        j                  ‰j                  «       d„ «       t        j                  ‰j                  «       d„ «       �n†t        j                  dˆˆˆˆfd„«       �nht        j                  ‰j                  t         j                  k(  xr ‰j                  t         j                  k(  d„ «       t        j                  ‰j                  «       dk(  xr ‰j                  «       dk(  ˆˆfd„«       ‰j                  d«      ‰k(  ru‰j                  d«      dk(  ra‰j                  d«      dk(  rM‰j                  d«      ‰k(  r9t        j                  ‰j                  «       xr ‰j                  «       d„ «       �nN‰j                  d«      ‰k(  rU‰j                  d«      ‰j                  d«      cxk(  rt        ‰d«      k(  r"n n‰j                  d«      t        ‰d«      k(  rnå‰j                  d«      ‰k(  rK‰j                  d«      ‰j                  d«      cxk(  rt        ‰d«      k(  rn n‰j                  d«      ‰k(  rn†‰j                  d«      t        ‰d«      k(  rK‰j                  d«      ‰j                  d«      cxk(  rt        ‰d«      k(  rn n‰j                  d«      ‰k(  rnt        j                  dˆˆˆˆˆfd„«       |�|n‰ j                  }t        j                  ‰ j                  d«      ‰j                  d«      |‰ j                   ¬«      S )Nc                 ó    — | t         j                  t         j                  t         j                  t         j                  t         j
                  fv S rD   ©rV   r•  Úfloat8_e5m2Úfloat8_e4m3fnuzÚfloat8_e5m2fnuzÚfloat4_e2m1fn_x2r‘   s    r8   Úis_fp8_or_fp4_typez2_check_scaled_mm_sizes.<locals>.is_fp8_or_fp4_type|  óA   € ØÜ×ÑÜ×ÑÜ×!Ñ!Ü×!Ñ!Ü×"Ñ"ð
ð 
ð 	
r:   rý   c                  óL   •— d‰j                  «       › d‰ j                  «       › �S ©Nz%Inputs must be 2D but got self.dim()=z and mat2.dim()=r�   ©r|  rš   s   €€r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>‡  ó'   ø€ Ð7¸¿¹»
°|ÐCSÐTX×T\ÑT\ÓT^ÐS_Ð`€ r:   c                  ó<   •— d‰j                   › d‰ j                   › �S ©Nz?Expected both inputs to be fp8 or fp4 types but got self.dtype=z and mat2.dtype=r‘   r  s   €€r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>‹  ó&   ø€ ÐQÐRV×R\ÑR\ÐQ]Ð]mÐnr×nxÑnxÐmyÐz€ r:   ré   r   rè   c                 ó,   — | d   | d   kD  xr | d   dk(  S rê  r4   rë  s    r8   Úis_row_majorz,_check_scaled_mm_sizes.<locals>.is_row_major”  ó"   € Ø˜!‘9˜v a™yÑ(Ò;¨V°A©Y¸!©^Ð;r:   c                 ó&   — | d   dk(  xr | d   dkD  S rê  r4   rë  s    r8   Úis_col_majorz,_check_scaled_mm_sizes.<locals>.is_col_major—  ó   € Ø˜!‘9 ‘>Ò3 f¨Q¡i°!¡mÐ3r:   c                 óV   — | j                  d«      dk(  xs | j                  d«      dk(  S rê  r¢   ©Ú	tensor_2ds    r8   Úhas_zero_dimz,_check_scaled_mm_sizes.<locals>.has_zero_dimš  ó)   € Ø—>‘> !Ó$¨Ñ)ÒC¨Y¯^©^¸AÓ->À!Ñ-CÐCr:   c                  ó*   •— d‰ j                  «       › �S ©Nz#self must be row_major, got stride rë  r±   s   €r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>   ó   ø€ Ð=¸d¿k¹k»m¸_ÐM€ r:   c                  ó*   •— d‰ j                  «       › �S ©Nz#mat2 must be col_major, got stride rë  ©r|  s   €r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>¤  r  r:   r3   rT  r   c                  ó,   •— d‰ j                  d«      › �S ©NzBExpected self.size(1) to be divisible by 16, but got self.size(1)=r3   r¢   r±   s   €r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>¨  ó   ø€ Ð\Ð]a×]fÑ]fÐghÓ]iÐ\jÐk€ r:   c                  ó"   •— d‰ j                   › �S ©Nz?Expected both dimensions of mat2 to be divisible by 16 but got r$  r  s   €r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>¬  ó   ø€ ÐYÐZ^×ZdÑZdÐYeÐf€ r:   c                   ó   — y)NzNFor tensorwise scaling, both scale_a and scale_b must be float (fp32) tensors.r4   r4   r:   r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>Ã  rm   r:   rU  ro  r5  c                   ó   — y)Nzscale_a must be contiguousr4   r4   r:   r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>å  rm   r:   c                   ó   — y)Nzscale_b must be contiguousr4   r4   r:   r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>é  rm   r:   Fc            	      óZ   •— d‰ › d‰j                  «       › d‰› d‰j                  «       › d�	S )NzTInvalid blockwise scaling configuration. For blockwise scaling, scale_a should have ú elements, got z, scale_b should have rÂ  rè  )Úexpected_a_sizeÚexpected_b_sizerò  ró  s   €€€€r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>î  sH   ø€ ðFØFUÐEVÐVeÐfm×fsÑfsÓfuÐevð w/Ø/>Ð.?¸ÈwÏ}É}ËÐN_Ð_`ðbð r:   c                   ó   — y)NzKFor rowwise scaling, both scale_a and scale_b must be float (fp32) tensors.r4   r4   r:   r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>÷  rm   r:   c                  óL   •— d‰ j                  «       ›d‰j                  «       ›�S )NzLFor non-tensorwise scaling, scale tensors must be 2D, but got scale_a.dim()=z and scale_b.dim()=r�   ©rò  ró  s   €€r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>ü  s,   ø€ ÐgÐY`×YdÑYdÓYfÐXhÐh|Ðnu×nyÑnyÓn{Ðm}Ð~€ r:   c                   ó   — y)Nz@Both scale_a and scale_b must be contiguous for rowwise scaling.r4   r4   r:   r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>  rm   r:   c                  ó¦  •— d‰› d‰› d‰› dt        ‰ d«      › d�	dt        ‰ d«      › dt        ‰d«      › d‰› dt        ‰ d«      › d�	z   dt        ‰ d«      › d‰› d	t        ‰d«      › dt        ‰ d«      › d�	z   dt        ‰ d«      › d‰› d
‰j                  d«      › d‰j                  d«      › d‰j                  d«      › d‰j                  d«      › d�z   S )Nú}Invalid scaling configuration. For tensorwise scaling, both scales should be scalar. For rowwise scaling, scale_a should be (ú, 1), scale_b should be (1, ú>). For (BlockWise1x128, BlockWise128x128), scale_a should be (rx   ro  ú), úscale_b should be (ú<). For (BlockWise1x128, BlockWise1x128), scale_a should be (z>). For (BlockWise128x128, BlockWise1x128), scale_a should be (ú). Got scale_a.size()=(r   r3   ú) and scale_b.size()=(ry   ©r9   r£   )Ú_kr˜  r  rò  ró  s   €€€€€r8   re   z(_check_scaled_mm_sizes.<locals>.<lambda>#  sL  ø€ ðCàCDÀ#ÐEaÐbcÐadð eVØVWÐUXÐXZÔ[cÐdfÐhkÓ[lÐZmÐmpðrð 0´¸¸SÓ0AÐ/BÀ"ÄXÈaÐQTÓEUÐDVð WTØTUÐSVÐVXÔYaÐbdÐfiÓYjÐXkÐknðpñ	pð 0´¸¸SÓ0AÐ/BÀ"ÀQÀCð HVÜV^Ð_`ÐbeÓVfÐUgÐgiÔjrÐsuÐwzÓj{Ði|Ð|ðAñAð 0´¸¸SÓ0AÐ/BÀ"ÀQÀCð H/Ø/6¯|©|¸A«Ð.?¸rÀ'Ç,Á,ÈqÃ/ÐARð S/Ø/6¯|©|¸A«Ð.?¸rÀ'Ç,Á,ÈqÃ/ÐARÐRSðUñ
Uð r:   ré  )rV   rg   r†   r^   rð   rÆ   r£   r™   Úfloat8_e8m0fnur•  r—   r’  rü  r9   r–  rŒ   rƒ   )rš   r|  rò  ró  rm  rô  ro  rõ  rý  r  r
  r  Úis_blockwise_scalingÚblock_size_kÚblock_size_mnÚnum_k_blocksÚpadded_num_k_blocksÚ
_out_dtyper2  r"  r#  r˜  r  s   ````              @@@@@r8   Ú_check_scaled_mm_sizesr:  r  sB  ÿø€ ò
ô 
‡L�LØ�‰‹
�a‰Ò+˜DŸH™H›J¨!™OÜ`ôô 
‡L�LÙ˜4Ÿ:™:Ó&ÒIÑ+=¸d¿j¹jÓ+IÜzôô 	�DÓ˜VÒ#Ü�tÓ Ò%Ü�tÓ Ó%ò	<ò	4ò	Dô �tÓ Ò%Ü�L‰LÙ˜TŸ[™[›]Ó+ÒA©|¸DÓ/AÛMôô �L‰LÙ˜TŸ[™[›]Ó+ÒA©|¸DÓ/AÛMôô �L‰LØ—	‘	˜!“˜rÑ! QÑ&Ûkôô �L‰LØ—	‘	˜!“˜rÑ! QÑ&ÒA¨4¯9©9°Q«<¸"Ñ+<ÀÑ+AÛfôð —
‘
‰ˆˆ2Ø�I‰I�a‹Lˆð —‘¤×!5Ñ!5Ñ5ò :Ø—M‘M¤U×%9Ñ%9Ñ9òð
 —‘¤×!4Ñ!4Ñ4ò 9Ø—M‘M¤U×%8Ñ%8Ñ8ð 	ð �=‰=‹?˜aÒ G§M¡M£O°qÒ$8ä�L‰LØ—‘¤§¡Ñ.ÒQ°7·=±=ÄEÇMÁMÑ3QÙhöò "ð �}‰}¤× 3Ñ 3Ò3ð  "�Ø˜!‘V‘à!�Ø—:‘:¤×!7Ñ!7Ò7Ø˜a™�BàˆMä# B¨Ó5ˆLÜ"*¨<¸Ó";¸aÑ"?Ðð ¤¨¨MÓ :Ñ:Ð=PÑPð ð ¤¨¨MÓ :Ñ:Ð=PÑPð ð
 —‘“ ?Ò2Ø—M‘M“O Ò6ä—‘Ø×)Ñ)Ó+Ù8ôô —‘Ø×)Ñ)Ó+Ù8öô
 —‘Øööô �L‰LØ—‘¤§¡Ñ.ÒQ°7·=±=ÄEÇMÁMÑ3QÙeôô
 �L‰LØ—‘“ Ñ"Ò9 w§{¡{£}¸Ñ'9Ü~ôð —‘˜Q“ 1Ò$Ø—L‘L “O qÒ(Ø—L‘L “O qÒ(Ø—L‘L “O qÒ(ô —‘Ø×)Ñ)Ó+ÒG°×0EÑ0EÓ0GÙ^öð
 —‘˜Q“ 1Ò$Ø—L‘L “O w§|¡|°A£ÔK¼(À2ÀsÓ:KÕKØ—L‘L “O¤x°°3Ó'7Ò7ð à—‘˜Q“ 1Ò$Ø—L‘L “O w§|¡|°A£ÔK¼(À2ÀsÓ:KÕKØ—L‘L “O qÒ(ð à—‘˜Q“¤8¨A¨sÓ#3Ò3Ø—L‘L “O w§|¡|°A£ÔK¼(À2ÀsÓ:KÕKØ—L‘L “O qÒ(ð ô —‘Ø÷ôð" (Ð3‘¸¿¹€JÜ�;‰;�t—y‘y “| T§Y¡Y¨q£\¸ÈDÏKÉKÔXÐXr:   c           
      ó&   — t        | |||||||«      S rD   )r:  )rš   r|  rò  ró  rm  rô  ro  rõ  s           r8   Úmeta_scaled_mmr<  6  s"   € ô "Øˆd�G˜W d¨L¸)À^óð r:   Úscale_recipe_aÚscale_recipe_bÚ	swizzle_aÚ	swizzle_bc           
      óä  ‡ ‡‡‡‡‡	‡‡‡ ‡!‡"‡#‡$‡%‡&— d„ }d„ }t        j                  ‰ j                  «       dk(  xr ‰j                  «       dk(  ˆˆ fd„«       t        j                   |‰ j                  «      xr  |‰j                  «      ˆˆ fd„«       ‰ j                  d   Š ‰ j                  d   Š‰j                  d   Š! |‰ j                  «      r |‰j                  «      rd}‰|z  Š|D �cg c]  }t        |«      ‘Œ }}|D �cg c]  }t        |«      ‘Œ }}‰r‰D �cg c]  }t        |«      ‘Œ c}Šnt        j                  gŠ‰	r‰	D �cg c]  }t        |«      ‘Œ c}Š	nt        j                  gŠ	t        ‰ «      dk(  st        ‰ «      d	k(  st        ‰ «      d
k(  �r
d„ }d„ }d„ }t        ‰ «      d
k7  rãt        j                   |‰ j                  «       «      xs  |‰ «      ˆ fd„«       t        j                   |‰j                  «       «      xs  |‰«      ˆfd„«       t        j                  ‰ j                  d«      dz  dk(  ˆ fd„«       t        j                  ‰j                  d«      dz  dk(  xr ‰j                  d«      dz  dk(  ˆfd„«       dt        t
           dt        t
           fd„}dt        t
           dt        t
           fd„}dt        t
           dt        t
           fd„}dt        t
           dt        t
           fd„}dt        t
           dt        t
           fd„}dt        t
           dt        t
           fd„}dt        t
           dt        t
           fd„}dt        t
           dt        t
           fd„} |||«      r‰t        j                  ‰d   j                  «       dk(  xrZ ‰d   j                  «       dk(  xrB ‰d   j                  t         j                  k(  xr  ‰d   j                  t         j                  k(  d„ «       �	nŽ |||«      r¥t        j                  ‰d   j                  d   ‰ k(  xrr ‰d   j                  «       ‰ k(  xrZ ‰d   j                  t         j                  k(  xr8 ‰d   j                  «       ‰!k(  xr  ‰d   j                  t         j                  k(  ˆˆˆˆ fd„«       �nà |||«      �rq‰d   j                  t         j                  k(  xr  ‰d   j                  t         j                  k(  }‰d   Š%‰%j                  d   ‰ k(  xrk ‰%j                  d   ‰dz  k(  xrT ‰%j                  d«      dk(  xr> ‰%j                  d«      ‰ k(  xs( ‰%j                  d   dk(  xr ‰%j                  d«      dk(  }‰d   Š&‰&j                  d   ‰!k(  xrk ‰&j                  d   ‰dz  k(  xrT ‰&j                  d«      dk(  xr> ‰&j                  d«      ‰!k(  xs( ‰&j                  d   dk(  xr ‰&j                  d«      dk(  }t        j                  |xr |xr |ˆˆ ˆ!ˆ%ˆ&fd „«       �ne |||«      �r�‰d   j                  t         j                  k(  xr  ‰d   j                  t         j                  k(  }t        ‰dz  d!«      Š‰d   Š%‰%j                  d   ‰k(  xrk ‰%j                  d   ‰ dz  k(  xrT ‰%j                  d«      dk(  xr> ‰%j                  d«      ‰k(  xs( ‰%j                  d   dk(  xr ‰%j                  d«      dk(  }‰d   Š&‰&j                  d   ‰!k(  xrk ‰&j                  d   ‰dz  k(  xrT ‰&j                  d«      dk(  xr> ‰&j                  d«      ‰!k(  xs( ‰&j                  d   dk(  xr ‰&j                  d«      dk(  }t        j                  |xr |xr |ˆˆˆ ˆ!ˆ%ˆ&fd"„«       �nÚ |||«      �r�‰d   j                  t         j                  k(  xr  ‰d   j                  t         j                  k(  }t        ‰dz  d!«      Š‰d   Š%‰%j                  d   ‰ k(  xrk ‰%j                  d   ‰dz  k(  xrT ‰%j                  d«      dk(  xr> ‰%j                  d«      ‰ k(  xs( ‰%j                  d   dk(  xr ‰%j                  d«      dk(  }‰d   Š&‰&j                  d   ‰k(  xrk ‰&j                  d   ‰!dz  k(  xrT ‰&j                  d«      dk(  xr> ‰&j                  d«      ‰k(  xs( ‰&j                  d   dk(  xr ‰&j                  d«      dk(  }t        j                  |xr |xr |ˆˆˆ ˆ!ˆ%ˆ&fd#„«       �nO |||«      �r¬t         j                  j                   rct#        ‰ j                  d   d$«      ‰ j                  d   z  Š"t#        ‰ j                  d   d$«      ‰ j                  d   z  Š#t        j                  Š$nŠt        ‰ j                  d   d«      t        t#        ‰ j                  d   d$«      d!«      z  Š"t        ‰j                  d   d«      t        t#        ‰ j                  d   d$«      d!«      z  Š#t        j$                  Š$t        j                  ‰d   j                  «       ‰"k(  xrn ‰d   j                  t         j&                  k(  xrL ‰d   j                  «       ‰#k(  xr4 ‰d   j                  t         j&                  k(  xr ‰d   ‰$k(  xr ‰	d   ‰$k(  ˆ"ˆ#ˆ$ˆˆˆˆ	fd%„«       �n™ |||«      rût        ‰ d«      t        t#        ‰d«      d!«      z  Š"t        ‰!d«      t        t#        ‰d«      d!«      z  Š#t        j$                  Š$t        j                  ‰d   j                  «       ‰"k(  xrn ‰d   j                  t         j(                  k(  xrL ‰d   j                  «       ‰#k(  xr4 ‰d   j                  t         j(                  k(  xr ‰d   ‰$k(  xr ‰	d   ‰$k(  ˆ"ˆ#ˆ$ˆˆˆˆ	fd&„«       �n• |||«      �rnt        ‰ d«      t        t#        ‰d«      d!«      z  Š"t        ‰!d«      t        t#        ‰d«      d!«      z  Š#t        j$                  Š$t        j                  ‰d   j                  «       ‰"k(  xrâ ‰d   j                  t         j(                  k(  xrÀ ‰d   j                  «       dk(  xr¨ ‰d   j                  t         j                  k(  xr† ‰d   j                  «       ‰#k(  xrn ‰d   j                  t         j(                  k(  xrL ‰d   j                  «       dk(  xr4 ‰d   j                  t         j                  k(  xr ‰d   ‰$k(  xr ‰	d   ‰$k(  ˆ"ˆ#ˆ$ˆˆˆˆ	fd'„«       nt        j                  d(ˆˆ ˆ!ˆˆfd)„«       |�|n‰ j                  }t        j*                  ‰ ‰!|‰ j,                  ¬*«      S c c}w c c}w c c}w c c}w )+Nc                 ó    — | t         j                  t         j                  t         j                  t         j                  t         j
                  fv S rD   rø  r‘   s    r8   rý  z5_check_scaled_mm_sizes_v2.<locals>.is_fp8_or_fp4_typeS  rþ  r:   c                 ó(   — | t         j                  k(  S rD   )rV   rü  r‘   s    r8   Úis_fp4_typez._check_scaled_mm_sizes_v2.<locals>.is_fp4_type\  s   € Øœ×.Ñ.Ñ.Ð.r:   rý   c                  óL   •— d‰j                  «       › d‰ j                  «       › �S r   r�   r  s   €€r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>a  r  r:   c                  ó<   •— d‰j                   › d‰ j                   › �S r  r‘   r  s   €€r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>e  r  r:   r   r3   ré   r   rè   c                 ó,   — | d   | d   kD  xr | d   dk(  S rê  r4   rë  s    r8   r  z/_check_scaled_mm_sizes_v2.<locals>.is_row_majorŠ  r  r:   c                 ó&   — | d   dk(  xr | d   dkD  S rê  r4   rë  s    r8   r
  z/_check_scaled_mm_sizes_v2.<locals>.is_col_major�  r  r:   c                 óV   — | j                  d«      dk(  xs | j                  d«      dk(  S rê  r¢   r  s    r8   r  z/_check_scaled_mm_sizes_v2.<locals>.has_zero_dim�  r  r:   c                  ó*   •— d‰ j                  «       › �S r  rë  r±   s   €r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>–  r  r:   c                  ó*   •— d‰ j                  «       › �S r  rë  r  s   €r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>š  r  r:   rT  c                  ó,   •— d‰ j                  d«      › �S r  r¢   r±   s   €r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>ž  r  r:   c                  ó"   •— d‰ j                   › �S r  r$  r  s   €r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>¢  r  r:   Úrecipe_aÚrecipe_bc                 óž   — t        | «      dk(  xr> t        |«      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S rð  )rÃ   r(   Ú
TensorWise©rN  rO  s     r8   Úis_tensorwisez0_check_scaled_mm_sizes_v2.<locals>.is_tensorwise¥  sT   € ä�H“ Ñ"ò :Ü˜“M QÑ&ò:à˜Q‘K¤;×#9Ñ#9Ñ9ò:ð ˜Q‘K¤;×#9Ñ#9Ñ9ð	r:   c                 óž   — t        | «      dk(  xr> t        |«      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S rð  )rÃ   r(   ÚRowWiserR  s     r8   Ú
is_rowwisez-_check_scaled_mm_sizes_v2.<locals>.is_rowwise­  sT   € ä�H“ Ñ"ò 7Ü˜“M QÑ&ò7à˜Q‘K¤;×#6Ñ#6Ñ6ò7ð ˜Q‘K¤;×#6Ñ#6Ñ6ð	r:   c                 óž   — t        | «      dk(  xr> t        |«      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S rð  )rÃ   r(   ÚBlockWise1x32rR  s     r8   Úis_mxz(_check_scaled_mm_sizes_v2.<locals>.is_mxµ  sT   € ä�H“ Ñ"ò =Ü˜“M QÑ&ò=à˜Q‘K¤;×#<Ñ#<Ñ<ò=ð ˜Q‘K¤;×#<Ñ#<Ñ<ð	r:   c                 óž   — t        | «      dk(  xr> t        |«      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S rð  )rÃ   r(   ÚBlockWise1x16rR  s     r8   Úis_nv_single_levelz5_check_scaled_mm_sizes_v2.<locals>.is_nv_single_level½  sV   € ô �H“ Ñ"ò =Ü˜“M QÑ&ò=à˜Q‘K¤;×#<Ñ#<Ñ<ò=ð ˜Q‘K¤;×#<Ñ#<Ñ<ð	r:   c                 óþ   — t        | «      dk(  xrn t        |«      dk(  xr^ | d   t        j                  k(  xrF | d   t        j                  k(  xr. |d   t        j                  k(  xr |d   t        j                  k(  S )Nrý   r   r3   )rÃ   r(   r[  rQ  rR  s     r8   Úis_nvz(_check_scaled_mm_sizes_v2.<locals>.is_nvÇ  s†   € ä�H“ Ñ"ò :Ü˜“M QÑ&ò:à˜Q‘K¤;×#<Ñ#<Ñ<ò:ð ˜Q‘K¤;×#9Ñ#9Ñ9ò:ð ˜Q‘K¤;×#<Ñ#<Ñ<ò	:ð
 ˜Q‘K¤;×#9Ñ#9Ñ9ðr:   c                 óž   — t        | «      dk(  xr> t        |«      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S rð  )rÃ   r(   ÚBlockWise1x128rR  s     r8   Úis_1x128_1x128z1_check_scaled_mm_sizes_v2.<locals>.is_1x128_1x128Ñ  sT   € ä�H“ Ñ"ò >Ü˜“M QÑ&ò>à˜Q‘K¤;×#=Ñ#=Ñ=ò>ð ˜Q‘K¤;×#=Ñ#=Ñ=ð	r:   c                 óž   — t        | «      dk(  xr> t        |«      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S rð  )rÃ   r(   r`  ÚBlockWise128x128rR  s     r8   Úis_1x128_128x128z3_check_scaled_mm_sizes_v2.<locals>.is_1x128_128x128Ù  sW   € ä�H“ Ñ"ò @Ü˜“M QÑ&ò@à˜Q‘K¤;×#=Ñ#=Ñ=ò@ð ˜Q‘K¤;×#?Ñ#?Ñ?ð	r:   c                 óž   — t        | «      dk(  xr> t        |«      dk(  xr. | d   t        j                  k(  xr |d   t        j                  k(  S rð  )rÃ   r(   rc  r`  rR  s     r8   Úis_128x128_1x128z3_check_scaled_mm_sizes_v2.<locals>.is_128x128_1x128á  sT   € ä�H“ Ñ"ò >Ü˜“M QÑ&ò>à˜Q‘K¤;×#?Ñ#?Ñ?ò>ð ˜Q‘K¤;×#=Ñ#=Ñ=ð	r:   c                   ó   — y)Nz\For Tensorwise scaling, both scale_a and scale_b must be single element float (fp32) tensorsr4   r4   r:   r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>ò  rm   r:   c            	      óš   •— d‰j                   d   › d‰d   j                  «       › d‰ j                   d   › d‰d   j                  «       › d�	S )Nz'For Rowwise scaling, scale_a must have r   z elements (got: z), and scale_b must have r3   ry   )r™   r—   )r|  rò  ró  rš   s   €€€€r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>û  sa   ø€ Ø=¸d¿j¹jÈ¹m¸_ÐL\Ð]dÐefÑ]g×]mÑ]mÓ]oÐ\pð q/Ø/3¯z©z¸!©}¨oÐ=MÈgÐVWÉj×N^ÑN^ÓN`ÐMaÐabðdð r:   ro  c                  ó¢   •— d‰› d‰ dz  › d‰j                   › d‰› d‰j                  › d‰› d‰ dz  › d‰j                   › d‰› d‰j                  › d�S )Nz>For 1x128 x 1x128 blockwise scaling, scale a must have shape [rx   ro  ú] (got: ú) and stride [1, ú)scale b must have shape [ry   ©r™   rÆ   )rŸ  rÒ  r¹  ÚsaÚsbs   €€€€€r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>  s~   ø€ ð0Ø01¨s°"°Q¸#±X°J¸hÀrÇxÁxÀjÐPaÐbcÐadÐdlÐmo×mvÑmvÐlwð x0Ø01¨s°"°Q¸#±X°J¸hÀrÇxÁxÀjÐPaÐbcÐadÐdlÐmo×mvÑmvÐlwÐwxðzð r:   r5  c                  ó¢   •— d‰› d‰dz  › d‰j                   › d‰› d‰j                  › d‰› d‰ dz  › d‰j                   › d‰› d‰j                  › d�S )Nz]For 128x128 x 1x128 blockwise scaling, L4 = {round_up(K / 128, 4)}, scale a must have shape [rx   ro  rj  rk  rl  ry   rm  ©rŸ  ÚL4rÒ  r¹  rn  ro  s   €€€€€€r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>7  s~   ø€ ð0Ø02¨t°2°a¸3±h°Z¸xÈÏÉÀzÐQbÐceÐbfÐfnÐoq×oxÑoxÐnyð z0Ø01¨s°"°Q¸#±X°J¸hÀrÇxÁxÀjÐPaÐbcÐadÐdlÐmo×mvÑmvÐlwÐwxðzð r:   c                  ó¢   •— d‰› d‰ dz  › d‰j                   › d‰› d‰j                  › d‰› d‰dz  › d‰j                   › d‰› d‰j                  › d�S )Nz]For 1x128 x 128x128 blockwise scaling, L4 = {round_up(K / 128, 4)}, scale a must have shape [rx   ro  rj  rk  rl  ry   rm  rq  s   €€€€€€r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>V  s~   ø€ ð0Ø01¨s°"°Q¸#±X°J¸hÀrÇxÁxÀjÐPaÐbcÐadÐdlÐmo×mvÑmvÐlwð x0Ø02¨t°2°a¸3±h°Z¸xÈÏÉÀzÐQbÐceÐbfÐfnÐoq×oxÑoxÐnyÐyzð|ð r:   rU  c                  óæ   •— d‰ › d‰d   j                  «       › d‰› d‰d   j                  «       › dt        j                  › d‰d   j                  › d‰d   j                  › d‰› d	‰d   › d‰d   › d
�S )Nz!for MX scaling scale_a must have ú (got: r   ú) and scale_b must have z). Scales must have types z (for self: ú	, mat_b: z) Must have swizzle type ú (got self: ry   )r—   rV   r3  r^   ©Úexpected_scale_a_elemsÚexpected_scale_b_elemsÚexpected_swizzlerò  ró  r?  r@  s   €€€€€€€r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>q  s°   ø€ Ø7Ð8NÐ7OÈwÐW^Ð_`ÑWa×WgÑWgÓWiÐVjð k-Ø-CÐ,DÀGÈGÐTUÉJ×L\ÑL\ÓL^ÐK_ð `"Ü"'×"6Ñ"6Ð!7°|ÀGÈAÁJ×DTÑDTÐCUÐU^Ð_fÐghÑ_i×_oÑ_oÐ^pð q.Ø.>Ð-?¸|ÈIÐVWÉLÈ>ÐYbÐclÐmnÑcoÐbpÐpqðsð r:   c                  ó„   •— d‰ › d‰d   j                  «       › d‰› d‰d   j                  «       › d‰› d‰d   › d‰d   › d�S )	Nz.for single-level NV scaling scale_a must have ru  r   rv  ú). Must have swizzle type rx  rw  ry   rè  ry  s   €€€€€€€r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>ƒ  sw   ø€ ØDÐE[ÐD\Ð\cÐdkÐlmÑdn×dtÑdtÓdvÐcwð x-Ø-CÐ,DÀGÈGÐTUÉJ×L\ÑL\ÓL^ÐK_ð `$Ø$4Ð#5°\À)ÈAÁ,ÀÈyÐYbÐcdÑYeÐXfÐfgðið r:   c                  ó„   •— d‰ › d‰d   j                  «       › d‰› d‰d   j                  «       › d‰› d‰d   › d‰d   › d�S )	Nz!for NV scaling scale_a must have ru  r   rv  r~  rx  rw  ry   rè  ry  s   €€€€€€€r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>˜  sv   ø€ Ø7Ð8NÐ7OÈwÐW^Ð_`ÑWa×WgÑWgÓWiÐVjð k-Ø-CÐ,DÀGÈGÐTUÉJ×L\ÑL\ÓL^ÐK_ð `$Ø$4Ð#5°\À)ÈAÁ,ÀÈyÐYbÐcdÑYeÐXfÐfgðið r:   Fc                  ób  •— d‰› d‰› d‰› dt        ‰ d«      › d�	dt        ‰ d«      › dt        ‰d«      › d‰› dt        ‰ d«      › d�	z   dt        ‰ d«      › d‰› d	‰d
   j                  d
«      › d‰d
   j                  d«      › d‰d
   j                  d
«      › d‰d
   j                  d«      › d�z   S )Nr)  r*  r+  rx   ro  r,  r-  r.  r/  r   r3   r0  ry   r1  )rŸ  rÒ  r¹  rò  ró  s   €€€€€r8   re   z+_check_scaled_mm_sizes_v2.<locals>.<lambda>¡  s  ø€ ð?à?@¸cÐA]Ð^_Ð]`ð aRØRSÐQTÐTVÔW_Ð`aÐcfÓWgÐVhÐhkðmð ,¬H°Q¸Ó,<Ð+=¸RÄÈÈCÓ@PÐ?Qð RPØPQÈsÐRTÔU]Ð^_ÐadÓUeÐTfÐfiðkñ	kð ,¬H°Q¸Ó,<Ð+=¸RÀ¸sð C+Ø+2°1©:¯?©?¸1Ó+=Ð*>¸bÀÈÁÇÁÐQRÓASÐ@Tð U+Ø+2°1©:¯?©?¸1Ó+=Ð*>¸bÀÈÁÇÁÐQRÓASÐ@TÐTUðWñWð r:   ré  )rV   rg   r†   r^   r™   r(   r)   Ú
NO_SWIZZLErð   rÆ   r£   rÄ   r—   r’  r?   rÖ  r×  r9   ÚSWIZZLE_32_4_4r3  r•  rŒ   rƒ   )'rš   r|  rò  r=  ró  r>  rm  ro  r?  r@  rõ  rý  rD  ÚK_packed_multiplierÚsir  r
  r  rS  rV  rY  r\  r^  ra  rd  rf  Útypes_okÚ
scale_a_okÚ
scale_b_okr9  rŸ  rr  rÒ  r¹  rz  r{  r|  rn  ro  s'   ``` `   ``                    @@@@@@@@@r8   Ú_check_scaled_mm_sizes_v2rˆ  F  s?  ÿþ€ ò
ò/ô 
‡L�LØ�‰‹
�a‰Ò+˜DŸH™H›J¨!™OÜ`ôô 
‡L�LÙ˜4Ÿ:™:Ó&ÒIÑ+=¸d¿j¹jÓ+IÜzôð 	�
‰
�1‰€AØ�
‰
�1‰€AØ�
‰
�1‰€Añ �4—:‘:Ô¡;¨t¯z©zÔ#:ØÐØ	Ð Ñ ˆà0>Ö?¨"”k "•oÐ?€NÐ?Ø0>Ö?¨"”k "•oÐ?€NÐ?áØ/8Ö9¨”[ •_Ò9‰	ô ×"Ñ"ð
ˆ	ñ Ø/8Ö9¨”[ •_Ò9‰	ô ×"Ñ"ð
ˆ	ô
 	�DÓ˜VÒ#Ü�tÓ Ò%Ü�tÓ Ó%ò	<ò	4ò	Dô �tÓ Ò%Ü�L‰LÙ˜TŸ[™[›]Ó+ÒA©|¸DÓ/AÛMôô �L‰LÙ˜TŸ[™[›]Ó+ÒA©|¸DÓ/AÛMôô �L‰LØ—	‘	˜!“˜rÑ! QÑ&Ûkôô �L‰LØ—	‘	˜!“˜rÑ! QÑ&ÒA¨4¯9©9°Q«<¸"Ñ+<ÀÑ+AÛfôð
	¤D¬Ñ$5ð 	ÄÄkÑARó 	ð	¤¤kÑ!2ð 	¼dÄ;Ñ>Oó 	ð	œD¤Ñ-ð 	¼¼kÑ9Jó 	ð	Üœ;Ñ'ð	Ü37¼Ñ3Dó	ð	œD¤Ñ-ð 	¼¼kÑ9Jó 	ð	¤T¬+Ñ%6ð 	Ä$Ä{ÑBSó 	ð	¤t¬KÑ'8ð 	ÄDÌÑDUó 	ð	¤t¬KÑ'8ð 	ÄDÌÑDUó 	ñ ˜¨Ô8ä�L‰LØ˜‘
× Ñ Ó" aÑ'ò 6Ø˜A‘J×$Ñ$Ó&¨!Ñ+ò6à˜A‘J×$Ñ$¬¯©Ñ5ò6ð ˜A‘J×$Ñ$¬¯©Ñ5Ùvöñ ˜¨Ô7Ü�L‰LØ˜‘
× Ñ  Ñ# qÑ(ò 6Ø˜A‘J×$Ñ$Ó&¨!Ñ+ò6à˜A‘J×$Ñ$¬¯©Ñ5ò6ð ˜A‘J×$Ñ$Ó&¨!Ñ+ò6ð ˜A‘J×$Ñ$¬¯©Ñ5öö
ñ ˜N¨NÕ;ð
 ˜‘
× Ñ ¤E§M¡MÑ1ÒW°g¸a±j×6FÑ6FÌ%Ï-É-Ñ6Wð ð ˜‘ˆBà—‘˜‘˜qÑ ò TØ—H‘H˜Q‘K 1¨¡8Ñ+òTà—I‘I˜a“L AÑ%òTð —Y‘Y˜q“\ QÑ&ÒR¨2¯8©8°A©;¸!Ñ+;Ò+QÀÇ	Á	È!ÃÐPQÑ@Qð	 ð ˜‘ˆBà—‘˜‘˜qÑ ò TØ—H‘H˜Q‘K 1¨¡8Ñ+òTà—I‘I˜a“L AÑ%òTð —Y‘Y˜q“\ QÑ&ÒR¨2¯8©8°A©;¸!Ñ+;Ò+QÀÇ	Á	È!ÃÐPQÑ@Qð	 ô �L‰LØÒ6˜ZÒ6¨J÷öñ ˜n¨nÕ=ð ˜‘
× Ñ ¤E§M¡MÑ1ÒW°g¸a±j×6FÑ6FÌ%Ï-É-Ñ6Wð ô ˜!˜c™' 1Ó%ˆBØ˜‘ˆBà—‘˜‘˜rÑ!ò UØ—H‘H˜Q‘K 1¨¡8Ñ+òUà—I‘I˜a“L AÑ%òUð —Y‘Y˜q“\ RÑ'ÒS¨B¯H©H°Q©K¸1Ñ,<Ò,RÀÇÁÈ1ÃÐQRÑARð	 ð ˜‘ˆBà—‘˜‘˜qÑ ò TØ—H‘H˜Q‘K 1¨¡8Ñ+òTà—I‘I˜a“L AÑ%òTð —Y‘Y˜q“\ QÑ&ÒR¨2¯8©8°A©;¸!Ñ+;Ò+QÀÇ	Á	È!ÃÐPQÑ@Qð	 ô �L‰LØÒ6˜ZÒ6¨J÷ð öñ ˜n¨nÕ=ð ˜‘
× Ñ ¤E§M¡MÑ1ÒW°g¸a±j×6FÑ6FÌ%Ï-É-Ñ6Wð ô ˜!˜c™' 1Ó%ˆBØ˜‘ˆBà—‘˜‘˜qÑ ò TØ—H‘H˜Q‘K 1¨¡8Ñ+òTà—I‘I˜a“L AÑ%òTð —Y‘Y˜q“\ QÑ&ÒR¨2¯8©8°A©;¸!Ñ+;Ò+QÀÇ	Á	È!ÃÐPQÑ@Qð	 ð ˜‘ˆBà—‘˜‘˜rÑ!ò UØ—H‘H˜Q‘K 1¨¡8Ñ+òUà—I‘I˜a“L AÑ%òUð —Y‘Y˜q“\ RÑ'ÒS¨B¯H©H°Q©K¸1Ñ,<Ò,RÀÇÁÈ1ÃÐQRÑARð	 ô �L‰LØÒ6˜ZÒ6¨J÷ð öñ �> >Õ2Ü�}‰}× Ò ä)1°$·*±*¸Q±-ÀÓ)DÀtÇzÁzÐRSÁ}Ñ)TÐ&Ü)1°$·*±*¸Q±-ÀÓ)DÀtÇzÁzÐRSÁ}Ñ)TÐ&Ü#.×#9Ñ#9Ñ ä)1°$·*±*¸Q±-ÀÓ)EÌÜ˜TŸZ™Z¨™]¨BÓ/°óIñ *Ð&ô *2°$·*±*¸Q±-ÀÓ)EÌÜ˜TŸZ™Z¨™]¨BÓ/°óIñ *Ð&ô $/×#=Ñ#=Ð Ü�L‰LØ˜‘
× Ñ Ó"Ð&<Ñ<ò 5Ø˜A‘J×$Ñ$¬×(<Ñ(<Ñ<ò5à˜A‘J×$Ñ$Ó&Ð*@Ñ@ò5ð ˜A‘J×$Ñ$¬×(<Ñ(<Ñ<ò5ð ˜a‘LÐ$4Ñ4ò	5ð
 ˜a‘LÐ$4Ñ4÷ñ öñ   °Ô?Ü%-¨a°Ó%5¼ÄÈ!ÈRÃÐRSÓ8TÑ%TÐ"Ü%-¨a°Ó%5¼ÄÈ!ÈRÃÐRSÓ8TÑ%TÐ"Ü*×9Ñ9ÐÜ�L‰LØ˜‘
× Ñ Ó"Ð&<Ñ<ò 5Ø˜A‘J×$Ñ$¬×(;Ñ(;Ñ;ò5à˜A‘J×$Ñ$Ó&Ð*@Ñ@ò5ð ˜A‘J×$Ñ$¬×(;Ñ(;Ñ;ò5ð ˜a‘LÐ$4Ñ4ò	5ð
 ˜a‘LÐ$4Ñ4÷ñ öñ �> >Õ2Ü%-¨a°Ó%5¼ÄÈ!ÈRÃÐRSÓ8TÑ%TÐ"Ü%-¨a°Ó%5¼ÄÈ!ÈRÃÐRSÓ8TÑ%TÐ"Ü*×9Ñ9ÐÜ�L‰LØ˜‘
× Ñ Ó"Ð&<Ñ<ò 	5Ø˜A‘J×$Ñ$¬×(;Ñ(;Ñ;ò	5à˜A‘J×$Ñ$Ó&¨!Ñ+ò	5ð ˜A‘J×$Ñ$¬¯©Ñ5ò	5ð ˜A‘J×$Ñ$Ó&Ð*@Ñ@ò		5ð
 ˜A‘J×$Ñ$¬×(;Ñ(;Ñ;ò	5ð ˜A‘J×$Ñ$Ó&¨!Ñ+ò	5ð ˜A‘J×$Ñ$¬¯©Ñ5ò	5ð ˜a‘LÐ$4Ñ4ò	5ð ˜a‘LÐ$4Ñ4÷ñ õô$ �L‰LØ÷
ôð (Ð3‘¸¿¹€JÜ�;‰;�q˜! :°d·k±kÔBÐBùòw	 @ùÚ?ùò :ùò :s   Ã0wÄw#Ä"w(Åw-r  Úcontraction_dimsc                 ó.   — t        | |||||||	|||¬«      S )N)rm  ro  r?  r@  rõ  )rˆ  )rš   r|  rò  r=  r?  ró  r>  r@  rm  r  r‰  rõ  s               r8   Úmeta_scaled_mm_v2r‹  ²  s2   € ô %ØØØØØØØØØØØ%ôð r:   c                 óZ   — t        | ||||d¬«       | j                  | j                  «      S ©NT)rC  r]  ©rš   r†   r“   r\  r�  r›  s         r8   Úmeta_scatter_reduce_twor�  Ð  s)   € ô �d˜C ¨¨VÀTÕJØ�>‰>˜$Ÿ*™*Ó%Ð%r:   c                 ó(   — t        | ||||d¬«       | S r�  ra  rŽ  s         r8   Úmeta_scatter_reduce__twor‘  ×  s   € ä�d˜C ¨¨VÀTÕJØ€Kr:   c                ót  ‡ — t        j                  d‰ j                  «       cxk  xr dk  nc ˆ fd„«       ‰ j                  «       dk(  r0t        j                  |t         j                  ‰ j
                  ¬«      S t        j                  ‰ j                  d«      |t         j                  ‰ j
                  ¬«      S )Nr   rý   c                  ó*   •— d‰ j                  «       › �S )NzAThe probability distributions dimensions must be 1 or 2, but got r�   r  s   €r8   re   z"meta_multinomial.<locals>.<lambda>â  s   ø€ ÐSÐTY×T]ÑT]ÓT_ÐS`Ða€ r:   r3   ré  )rV   rg   r†   rŒ   r•   rƒ   r£   )rØ   Únum_samplesÚreplacementr
  s   `   r8   Úmeta_multinomialr–  Ý  s|   ø€ ô 
‡L�LØ	ˆE�I‰I‹KÖ˜1ÔÛaôð ‡y�yƒ{�aÒÜ�{‰{˜;¬e¯j©jÀÇÁÔNÐNÜ�;‰;Ø�
‰
�1‹�{¬%¯*©*¸U¿\¹\ôð r:   c                 ó"   — d}| D ]  }||z  }Œ	 |S r2   r4   )Úvsr�  Úvs      r8   Úmultiply_integersrš  ë  s$   € Ø	€AØò ˆØ	ˆQ‰‰ðà€Hr:   c                 óL  ‡ ‡‡‡— t        j                  t        ‰«      ‰k(  ˆˆfd„«       ‰dz   Št        j                  t        ‰ «      ‰k(  ˆˆ fd„«       t        j                  t        d„ ‰ dd  D «       «      xr t        d„ ‰D «       «      ˆ ˆfd„«       ‰ d d \  }}||g‰¢­S )Nc                  ó&   •— d‰ › dt        ‰«      › �S )Nz%It is expected output_size equals to ú, but got size rø  )Únum_spatial_dimsrª  s   €€r8   re   z'upsample_common_check.<locals>.<lambda>õ  s   ø€ Ð7Ð8HÐ7IÈÔY\Ð]hÓYiÐXjÐk€ r:   rý   c                  ó&   •— d‰ › dt        ‰«      › �S )Nz$It is expected input_size equals to r�  rø  )Úexpected_input_dimsr€  s   €€r8   re   z'upsample_common_check.<locals>.<lambda>ú  s   ø€ Ð6Ð7JÐ6KÈ?Ô[^Ð_iÓ[jÐZkÐl€ r:   c              3   ó&   K  — | ]	  }|d kD  –— Œ y­w©r   Nr4   ©rr   rØ  s     r8   rt   z(upsample_common_check.<locals>.<genexpr>þ  s   è ø€ Ò*�aˆA��EÑ*ùó   ‚c              3   ó&   K  — | ]	  }|d kD  –— Œ y­wr¢  r4   r£  s     r8   rt   z(upsample_common_check.<locals>.<genexpr>þ  s   è ø€ Ò2N¸Q°1°qµ5Ñ2Nùr¤  c                  ó   •— d‰ › d‰› �S )NzDInput and output sizes should be greater than 0, but got input size z and output size r4   )r€  rª  s   €€r8   re   z'upsample_common_check.<locals>.<lambda>ÿ  s   ø€ ð Ø �\Ð!2°;°-ðA€ r:   )rV   rg   rÃ   rB  )r€  rª  rž  r  Úchannelsr   s   ```  @r8   Úupsample_common_checkr¨  ò  s£   û€ Ü	‡L�LÜˆKÓÐ,Ñ,Ükôð +¨QÑ.ÐÜ	‡L�LÜˆJ‹Ð.Ñ.Ülôô
 
‡L�LÜÑ*˜: a b˜>Ô*Ó*ÒN¬sÑ2NÀ+Ô2NÓ/Nô	Aôð " " 1�~Ñ€FˆHØ�HÐ+˜{Ñ+Ð+r:   c                 ó4  ‡ — t        j                  ‰ j                  «       dk7  xs t        ‰ j	                  «       dd  «      ˆ fd„«       t        ‰ j	                  «       |d¬«      }‰ j                  |«      j                  t        j                  ‰ «      ¬«      S )Nr   r3   c                  ó*   •— d‰ j                  «       › �S )Nz>Non-empty 3D data tensor expected but got a tensor with sizes r¢   r  s   €r8   re   z$upsample_nearest1d.<locals>.<lambda>  ó   ø€ ÐPÐQV×Q[ÑQ[ÓQ]ÐP^Ð_€ r:   ©rž  r«   ©
rV   rg   r—   rš  r£   r¨  r˜   rY  rO   r   )rØ   rª  ÚscalesÚfull_output_sizes   `   r8   Úupsample_nearest1dr°    ó†   ø€ ô 
‡L�LØ�‰‹˜ÑÒAÔ/°·
±
³¸Q¸RÐ0@ÓAÛ_ôô -Ø�
‰
‹�k°AôÐð �?‰?Ð+Ó,×/Ñ/Ü×1Ñ1°%Ó8ð 0ó ð r:   c                 ó¾  ‡ — t        j                  ‰ j                  «       dk7  xs t        ‰ j	                  «       dd  «      ˆ fd„«       t        ‰ j	                  «       |d¬«      }‰ j                  |«      }t        j                  ‰ «      }‰ j                  \  }}}}‰ j                  j                  dk(  r|dk  rt         j                  }|j                  |¬«      }|S )	Nr   r3   c                  ó*   •— d‰ j                  «       › �S ©Nz>Non-empty 4D data tensor expected but got a tensor with sizes r¢   r  s   €r8   re   z$upsample_nearest2d.<locals>.<lambda>  r«  r:   rý   r¬  ré   r5  r«   )rV   rg   r—   rš  r£   r¨  r˜   rO   r   r™   rƒ   rz   r°   Ú
contiguous)	rØ   rª  Úscales_hÚscales_wr¯  r÷   r¬   rR   Ú
n_channelss	   `        r8   Úupsample_nearest2dr¹    sÆ   ø€ ô 
‡L�LØ�‰‹˜ÑÒAÔ/°·
±
³¸Q¸RÐ0@ÓAÛ_ôô -Ø�
‰
‹�k°AôÐð �_‰_Ð-Ó.€Fô ×/Ñ/°Ó6€Mð  Ÿ+™+Ñ€A€z�1�aØ‡|�|×Ñ˜FÒ" z°A¢~Ü×/Ñ/ˆà×Ñ¨]ÐÓ;€Fà€Mr:   rª  r€  r¶  r·  c                 óX  ‡ ‡‡— t        ||d¬«      Št        j                  ‰ j                  dk(  ˆ fd„«       t	        d«      D ]2  Št        j                  ‰ j                  ‰«      ‰‰   k(  ˆˆ ˆfd„«       Œ4 ‰ j                  |«      j                  t        j                  ‰ «      ¬«      S )Nrý   r¬  r5  c                  ó"   •— d‰ j                   › �S ©NzFExpected grad_output to be a tensor of dimension 4 but got: dimension r7  rÐ  s   €r8   re   z-upsample_nearest2d_backward.<locals>.<lambda>C  ó   ø€ ÐXÐYd×YiÑYiÐXjÐk€ r:   c            
      óD   •— d‰› d‰ ‰   › d‰› d‰j                  ‰«      › �S )NzCExpected grad_output to have the same shape as output; output.size(ú) = z but got grad_output.size(r¢   ©r¯  r*  rÜ   s   €€€r8   re   z-upsample_nearest2d_backward.<locals>.<lambda>H  s>   ø€ ð Ø !˜s $Ð'7¸Ñ':Ð&;Ø,¨Q¨C¨t°K×4DÑ4DÀQÓ4GÐ3HðJð r:   r«   )
r¨  rV   rg   r¥   rÅ   r£   r˜   rY  rO   r   )r*  rª  r€  r¶  r·  r¯  rÜ   s   `    @@r8   Úupsample_nearest2d_backwardrÁ  1  s¨   ú€ ô -Ø�K°!ôÐô 
‡L�LØ×Ñ˜AÑÛkôô �1‹Xò 
ˆÜ�‰Ø×Ñ˜QÓÐ#3°AÑ#6Ñ6õõ	
ð
ð × Ñ  Ó,×/Ñ/Ü×1Ñ1°+Ó>ð 0ó ð r:   c                 ó4  ‡ — t        j                  ‰ j                  «       dk7  xs t        ‰ j	                  «       dd  «      ˆ fd„«       t        ‰ j	                  «       |d¬«      }‰ j                  |«      j                  t        j                  ‰ «      ¬«      S )Nr   r3   c                  ó*   •— d‰ j                  «       › �S )Nz>Non-empty 5D data tensor expected but got a tensor with sizes r¢   r  s   €r8   re   z$upsample_nearest3d.<locals>.<lambda>Z  r«  r:   r0   r¬  r«   r­  )rØ   rª  Úscales_dr¶  r·  r¯  s   `     r8   Úupsample_nearest3drÅ  T  r±  r:   c                 ó  — t        j                  | «      t        j                  | t         j                  ¬«      }}|�Æ|�Ät        |t        «      st        dt        |«      › �«      ‚t        |t        «      st        dt        |«      › �«      ‚|j                  }|j                  «       }	t        ||«      }t        ||«      }|j                  ||	«       |j                  ||	«       t        ||¬«       t        ||¬«       ||fS ||fS )Nr‘   zvalues must be TensorLike, got z indices must be TensorLike, got )r¼  r½  )rV   r¯   r·   rp   r   rŠ   rz   r™   rÆ   r!   rË   r#   )
rš   Ústabler†   Ú
descendingr³   r´   r™  rÜ   r¨   Ú
out_strides
             r8   Ú	meta_sortrÊ  d  së   € ô ×Ñ˜DÓ!¤5×#3Ñ#3°DÄÇÁÔ#L€q€AØÐ˜gÐ1Ü˜&¤*Ô-Ü Ð#BÄ4ÈÃ<À.Ð!QÓRÐRÜ˜'¤:Ô.Ü Ð#CÄDÈÃMÀ?Ð!SÓTÐTð —G‘Gˆ	Ø—X‘X“Zˆ
Ü" 6¨9Ó5ˆÜ# G¨YÓ7ˆØ×Ñ˜9 jÔ1Ø×Ñ˜I zÔ2Ü ¨FÕ3Ü ¨GÕ4Ø�wˆÐØˆaˆ4€Kr:   c           	      ó  ‡ ‡‡‡‡‡‡‡— t        j                  ‰ j                  dk(  ˆ fd„«       t        j                  ‰ j                  ‰j                  k(  ˆˆ fd„«       ‰ j	                  d«      Š‰�‚t        j                  ‰j                  dk(  ˆfd„«       t        j                  ‰j                  «       ‰k(  ˆˆfd„«       t        j                  ‰j                  ‰j                  k(  ˆˆfd„«       t        j                  ‰j                  dk(  ˆfd„«       ‰ j	                  d	«      ‰z  ‰z  Št        j                  ‰j                  «       ‰k(  ˆˆˆˆ ˆfd
„«       t        j                  t        ˆ fd„‰‰‰‰fD «       «      d„ «       y )Nrý   c                  ó"   •— ‰ j                   › d�S ©Nz != 2r7  )Úinput_gatess   €r8   re   z%rnn_cell_checkSizes.<locals>.<lambda>‰  ó   ø€ °;×3CÑ3CÐ2DÀEÐ0J€ r:   c                  ó:   •— ‰j                   › d‰ j                   › �S ©Nro  r$  )Úhidden_gatesrÎ  s   €€r8   re   z%rnn_cell_checkSizes.<locals>.<lambda>Œ  s    ø€ �;×$Ñ$Ð% T¨,×*<Ñ*<Ð)=Ð>€ r:   r3   c                  ó"   •— ‰ j                   › d�S )Nz != 1r7  )Ú
input_biass   €r8   re   z%rnn_cell_checkSizes.<locals>.<lambda>�  s   ø€ °j·o±oÐ5FÀeÐ3L€ r:   c                  ó.   •— ‰j                  «       › d‰ › �S rÑ  rè  )Ú
gates_sizerÔ  s   €€r8   re   z%rnn_cell_checkSizes.<locals>.<lambda>“  s   ø€ �z×'Ñ'Ó)Ð*¨$¨z¨lÐ;€ r:   c                  ó:   •— ‰j                   › d‰ j                   › �S rÑ  r$  )Úhidden_biasrÔ  s   €€r8   re   z%rnn_cell_checkSizes.<locals>.<lambda>—  s    ø€ �z×'Ñ'Ð(¨¨[×->Ñ->Ð,?Ð@€ r:   c                  ó"   •— ‰ j                   › d�S rÍ  r7  )Úprev_hiddens   €r8   re   z%rnn_cell_checkSizes.<locals>.<lambda>™  rÏ  r:   r   c            
      ó`   •— ‰j                  «       › d‰j                  d«      › d‰› d‰› d‰ › d�
S )Nro  r   z * z // z (aka ry   )r—   r£   )Úexpected_prev_hidden_numelÚfactorrÖ  rÎ  rÚ  s   €€€€€r8   re   z%rnn_cell_checkSizes.<locals>.<lambda>�  sB   ø€ �;×$Ñ$Ó&Ð' t¨K×,<Ñ,<¸QÓ,?Ð+@ÀÀJÀ<ÈtÐTZÐS[Ð[aÐb|Ða}Ð}~Ð€ r:   c              3   óP   •K  — | ]  }|j                   ‰j                   k(  –— Œ y ­wrD   r7  )rr   r=   rÎ  s     €r8   rt   z&rnn_cell_checkSizes.<locals>.<genexpr>   s(   øè ø€ ò 
àð �H‰H˜×*Ñ*Õ*ñ
ùs   ƒ#&c                   ó   — y)Nz%expected all inputs to be same devicer4   r4   r:   r8   re   z%rnn_cell_checkSizes.<locals>.<lambda>¤  rm   r:   )rV   rg   r¥   r™   r£   r—   rB  )rÎ  rÒ  rÔ  rØ  rÝ  rÚ  rÜ  rÖ  s   ``````@@r8   Úrnn_cell_checkSizesrà  �  s@  ÿ€ ô 
‡L�L�×!Ñ! QÑ&Ó(JÔKÜ	‡L�LØ×Ñ˜\×/Ñ/Ñ/Ü>ôð ×!Ñ! !Ó$€JØÐÜ�‰�Z—_‘_¨Ñ)Ó+LÔMÜ�‰Ø×ÑÓ *Ñ,Ü;ô	
ô 	�‰Ø×Ñ × 1Ñ 1Ñ1Ü@ô	
ô 
‡L�L�×!Ñ! QÑ&Ó(JÔKØ!,×!1Ñ!1°!Ó!4°zÑ!AÀVÑ!KÐÜ	‡L�LØ×ÑÓÐ9Ñ9ßôô 
‡L�LÜó 
à" J°¸[ÐIô
ó 	
ñ 	8õr:   c                 ó
  — t        | |||d|«       t        j                  | t        j                  ¬«      }t        j                  |t        j                  ¬«      }t        j                  |t        j                  ¬«      }|||fS )Nr5  r«   )rà  rV   r¯   r°   )rÎ  rÒ  ÚcxrÔ  rØ  Ú	workspaceÚhyÚcys           r8   Ú_thnn_fused_lstm_cell_metaræ  ¨  sk   € ô ˜ \°:¸{ÈAÈrÔRÜ× Ñ  ¼E×<SÑ<SÔT€IÜ	×	Ñ	˜"¬E×,CÑ,CÔ	D€BÜ	×	Ñ	˜"¬E×,CÑ,CÔ	D€BØ��IÐÐr:   c                 ób  — t        |«      dk7  }|r t        |«      }|d   }| j                  d   }nB|
r| j                  d   n| j                  d   }|
r| j                  d   n| j                  d   }d}|rdnd}|dk7  r|n|}|r|||z  g}n|
r||||z  gn||||z  g}| j                  |«      }|	|z  ||g}|€"t        j                  d| j
                  ¬«      }n|j                  |«      }|j                  |	|z  ||g«      }|rdnd}| j                  |t        j                  ¬«      }|||||fS )Nr   r3   rœ   rý   r7  r‘   )rÃ   r™   r˜   rV   rŒ   rƒ   r  )rØ   rk  Úweight_stride0Ú
weight_bufÚhxrâ  rŒ  Úhidden_sizeÚ	proj_sizeÚ
num_layersÚbatch_firstÚdropoutÚtrainÚbidirectionalr*  Údropout_stateÚis_input_packedÚ
seq_lengthÚ
mini_batchÚbatch_sizes_sumÚnum_directionsÚout_sizer¨   r÷   Ú
cell_shaperå  rä  Úreserve_shapeÚreserves                                r8   Ú
_cudnn_rnnrü  ·  sS  € ô& ˜+Ó&¨!Ñ+€OÙÜ˜Ó%ˆ
Ø  ‘^ˆ
ØŸ+™+ a™.‰á'2�U—[‘[ ’^¸¿¹ÀA¹ˆ
Ù'2�U—[‘[ ’^¸¿¹ÀA¹ˆ
Øˆá'‘Q¨Q€NØ%¨šN‰y°€HÙØ$ h°Ñ&?Ð@‰	ñ ð ˜ X°Ñ%>Ñ?à˜j¨(°^Ñ*CÐDð 	ð
 �_‰_˜YÓ'€Fà˜~Ñ-¨z¸;ÐG€JØ	€zÜ�[‰[˜ 5§<¡<Ô0‰à�\‰\˜*Ó%ˆà	�‰�z NÑ2°JÀÐIÓ	J€Bñ ‘A A€MØ�o‰o˜m´5·;±;ˆoÓ?€Gà�2�r˜7 JÐ.Ð.r:   c                 óº   — d}|D ])  }|j                  «       dkD  sŒ||j                  «       z  }Œ+ | j                  |f«      }t        | |||||||d|||	|
|||«      S r¥  )r—   r˜   rü  )rØ   rk  rè  rê  râ  rŒ  rë  rí  rî  rï  rð  rñ  r*  rò  Útotal_weight_elemsr  ré  s                    r8   Ú
miopen_rnnrÿ  ï  sˆ   € ð& ÐØò ,ˆØ�7‰7‹9�q‹=Ø !§'¡'£)Ñ+Ñð,ð —‘Ð"4Ð!6Ó7€JÜØØØØØ
Ø
ØØØ	ØØØØØØØó!ð r:   c                 ó(  — |r| j                   d   n| j                   d   }|r| j                   d   n| j                   d   }|
}|r|||gn|||g}| j                  |«      }|€"t        j                  d| j                  ¬«      }n|j                  |j                   «      }|€"t        j                  d| j                  ¬«      }n|j                  |j                   «      }t        j                  d| j                  t        j
                  ¬«      }||||fS )Nr3   r   r7  r¶   )r™   r˜   rV   rŒ   rƒ   r  )rØ   Úw0Úw1Úw2Úw3Úhx_Úcx_rÂ   r*  rŒ  rë  rí  Ú
has_biasesrñ  rî  rð  rô  rõ  Úoutput_chanelsr¨   r÷   rä  rå  rã  s                           r8   Úmkldnn_rnn_layerr		    sê   € ñ& $/�—‘˜Q’°E·K±KÀ±N€JÙ#.�—‘˜Q’°E·K±KÀ±N€JØ €Nñ ð 
�Z Ñ0à˜* nÐ5ð ð
 �_‰_˜YÓ'€FØ
€{Ü�[‰[˜ 5§<¡<Ô0‰à�]‰]˜3Ÿ9™9Ó%ˆØ
€{Ü�[‰[˜ 5§<¡<Ô0‰à�]‰]˜3Ÿ9™9Ó%ˆÜ—‘˜A e§l¡l¼%¿+¹+ÔF€IØ�2�r˜9Ð$Ð$r:   c                 óÈ   ‡‡— | j                   dk(  r%t        j                  ‰dk(  xs ‰dk(  ˆˆfd„«       y t        j                  | j                  ‰«      dk7  ˆˆfd„«       y )Nr   rœ   c                  ó   •— ‰› d‰ › �S )Nz4: Expected reduction dim -1 or 0 for scalar but got r4   ©r†   r  s   €€r8   re   z'zero_numel_check_dims.<locals>.<lambda>H  s   ø€ �w�iÐSÐTWÐSXÐY€ r:   c                  ó   •— ‰› d‰ › d�S )Nz: Expected reduction dim z to have non-zero size.r4   r	  s   €€r8   re   z'zero_numel_check_dims.<locals>.<lambda>M  s   ø€ �w�iÐ8¸¸Ð=TÐU€ r:   )r¥   rV   r–   r£   )rš   r†   r  s    ``r8   Úzero_numel_check_dimsr	  D  sR   ù€ Ø‡y�y�A‚~Ü×ÑØ�1‰HÒ!˜˜r™	ÜYõ	
ô
 	×ÑØ�I‰I�c‹N˜aÑÜUõ	
r:   c                 ó®   ‡ — |�(t        ||j                  «       «      }t        ||‰ «       y t        j                  |j                  «       dk7  ˆ fd„«       y )Nr   c                  ó   •— ‰ › d�S )Nz@: Expected reduction dim to be specified for input.numel() == 0.r4   rV  s   €r8   re   z%check_argmax_argmin.<locals>.<lambda>Y  s   ø€ �t�fÐ\Ð]€ r:   )r¸   r†   r	  rV   rg   r—   )r÷  rš   r†   s   `  r8   Úcheck_argmax_argminr	  R  sC   ø€ Ø
€Ü˜S $§(¡(£*Ó-ˆÜ˜d C¨Õ.ä�‰Ø�J‰J‹L˜AÑÛ]õ	
r:   c                 óÂ   — t        d| |«       t        j                  | j                  |�|fnd «      }t	        | ||«      }| j                  |t        j                  ¬«      S )NÚargmaxr‘   )r	  rO   r¼  r™   r½  r˜   rV   r·   )rš   r†   r¿  r¾  r™   s        r8   Úargmax_argmin_metar	  ]  sQ   € ä˜ $¨Ô,Ü×Ñ §
¡
°c°o¨S©FÈ4ÓP€DÜ$ T¨4°Ó9€EØ�>‰>˜%¤u§{¡{ˆ>Ó3Ð3r:   c                 ó|   — |t         j                  k(  rt         j                  }t        j                  d||||¬«      S )Nr4   r  )rV   ÚjaggedrØ  rŒ   )rØ  r^   r‚   rƒ   r„   s        r8   Úscalar_tensorr	  e  s5   € ð
 ”—‘ÒÜ—‘ˆÜ�;‰;Ø
�% ¨vÀ*ôð r:   c                 óž  — t        || j                  «       d¬«      }| j                  «       dk(  rdn| j                  |«      }t        j                  |dk\  «       t        j                  ||k  d„ «       t        | j                  «      }t        |«      dkD  r|||<   | j                  |«      | j                  |t        j                  ¬«      fS )NT©r$  r   r3   c                   ó   — y)Nzk not in range for dimensionr4   r4   r:   r8   re   ztopk_meta.<locals>.<lambda>w  rm   r:   r‘   )
r¸   r†   r£   rV   rg   rÄ   r™   rÃ   r˜   r·   )rš   rw  r†   Úlargesträ   Ú	sliceSizeÚtopKSizes          r8   Ú	topk_metar	  q  sž   € ô ˜˜dŸh™h›j°dÔ
;€CØ—X‘X“Z 1’_‘¨$¯)©)°C«.€IÜ	‡L�L��a‘ÔÜ	‡L�L��i‘Ñ!GÔHä�D—J‘JÓ€HÜ
ˆ8ƒ}�qÒØˆ�‰Ø�>‰>˜(Ó# T§^¡^°HÄEÇKÁK ^Ó%PÐPÐPr:   c                 óÎ   — |€|€t        d«      ‚|j                  «       }| j                  «       }	t        j                  ||	j                  |	j
                  |	j                  ¬«      S )Nz;segment_reduce(): Either lengths or offsets must be defined)r^   rƒ   r‚   )rŠ   rµ  rV   r¯   r^   rƒ   r‚   )
rx  r÷   r/  r�  r¨  r©  rª  r¬  Údata_contigÚgrad_contigs
             r8   Úmeta__segment_reduce_backwardr"	    sh   € ð
 €˜7˜?ÜØIó
ð 	
ð —/‘/Ó#€KØ—/‘/Ó#€KÜ×ÑØØ×ÑØ×!Ñ!Ø×!Ñ!ô	ð r:   c                 óè  ‡— ddl m} t        ‰| j                  «       d¬«      Š| j                  «       dkD  r| j	                  ‰«      nd}t        j                   ||dk\  ||k  «      ˆfd„«       t        | j                  d ‰ | j                  ‰dz   d  z   «      }|r%| j                  «       dkD  r|j                  ‰d«       | j                  |«      | j                  |t
        j                  ¬«      fS )Nr   )rf  Tr	  r3   c                  ó   •— d‰ › �S )Nz9kthvalue(): selected number k out of range for dimension r4   r�   s   €r8   re   zkthvalue_meta.<locals>.<lambda>›  s   ø€ ÐKÈCÈ5ÐQ€ r:   r‘   )rV  rf  r¸   r†   r£   rV   rg   rÄ   r™   ra  r˜   r·   )rš   rw  r†   r¿  rf  ÚdimSizer™   s     `    r8   Úkthvalue_metar&	  ’  sÅ   ø€ õ >ä
˜˜dŸh™h›j°dÔ
;€CØ $§¡£
¨Q¢ˆd�i‰i˜Œn°A€GÜ	‡L�LÙ��Q‘˜˜W™Ó%ÛQôô
 �—‘˜D˜SÐ! D§J¡J¨s°Q©w¨yÐ$9Ñ9Ó:€EÙ�4—8‘8“: ’>Ø�‰�S˜!ÔØ�>‰>˜%Ó  $§.¡.°¼e¿k¹k .Ó"JÐJÐJr:   c                 ó€  — | �| n|}t        j                  |j                  «       dk(  d„ «       |j                  «       }| �(t        j                  | j                  «       |k(  d„ «       |�(t        j                  |j                  «       |k(  d„ «       t        j                  |j                  «       |k(  d„ «       t        j                  |j                  «       |k(  d„ «       t        j                  |j                  «       dk(  d„ «       t        j                  |j	                  «       |d   |d	   z  d
z  k(  d„ «       y )Nrý   c                   ó   — y©NÚ r4   r4   r:   r8   re   z(checkLSTMBackwardSizes.<locals>.<lambda>ª  rm   r:   c                   ó   — yr)	  r4   r4   r:   r8   re   z(checkLSTMBackwardSizes.<locals>.<lambda>­  rm   r:   c                   ó   — yr)	  r4   r4   r:   r8   re   z(checkLSTMBackwardSizes.<locals>.<lambda>¯  rm   r:   c                   ó   — yr)	  r4   r4   r:   r8   re   z(checkLSTMBackwardSizes.<locals>.<lambda>°  rm   r:   c                   ó   — yr)	  r4   r4   r:   r8   re   z(checkLSTMBackwardSizes.<locals>.<lambda>±  rm   r:   c                   ó   — yr)	  r4   r4   r:   r8   re   z(checkLSTMBackwardSizes.<locals>.<lambda>²  rm   r:   r   r3   r5  c                   ó   — yr)	  r4   r4   r:   r8   re   z(checkLSTMBackwardSizes.<locals>.<lambda>³  rm   r:   )rV   rg   r†   r£   r—   )Úgrad_hyÚgrad_cyrâ  rå  rã  Údefined_gradÚexp_sizes          r8   ÚcheckLSTMBackwardSizesr5	  ¨  sí   € Ø%Ð1‘7°w€LÜ	‡L�L�×!Ñ!Ó# qÑ(©*Ô5Ø× Ñ Ó"€HØÐÜ�‰�W—\‘\“^ xÑ/±Ô<ØÐÜ�‰�W—\‘\“^ xÑ/±Ô<Ü	‡L�L�—‘“˜hÑ&©
Ô3Ü	‡L�L�—‘“˜hÑ&©
Ô3Ü	‡L�L�—‘“ AÑ%¡zÔ2Ü	‡L�L�—‘Ó" h¨q¡k°H¸Q±KÑ&?À!Ñ&CÑCÁZÕPr:   c                 óÎ   — | €|€yt        | ||||«       t        j                  |t        ¬«      }t        j                  |t        ¬«      }|r|j	                  dd¬«      nd }|||fS )N©NNNr«   r   F)r¿  )r5	  rV   r¯   Úlegacy_contiguous_memory_formatr  )	r1	  r2	  râ  rå  rã  Úhas_biasÚ
grad_gatesÚgrad_cxrÅ  s	            r8   Ú#_thnn_fused_lstm_cell_backward_implr<	  ·  sl   € à€˜7˜?ØÜ˜7 G¨R°°YÔ?Ü×!Ñ!ØÔ!@ô€Jô ×Ñ˜rÔ1PÔQ€GÙ4<�
—‘˜q¨%�Ô0À$€IØ�w 	Ð)Ð)r:   c                 ó  — d }d }d }|d   r|j                  | j                  «       «      }|d   s|d   rQ|j                  |j                  d«      | j                  d«      f«      }|j                  |j                  d«      «      }|||fS )Nr   r3   rý   rœ   r'  )r  r  r  r  r-  Úgrad_weightrÅ  s          r8   Úlinear_backwardr?	  Å  sŒ   € à€JØ€KØ€IØ�1‚~Ø!×+Ñ+¨F¯K©K«MÓ:ˆ
Ø�1‚~˜ QšØ"×,Ñ,¨l×.?Ñ.?ÀÓ.CÀVÇ[Á[ÐQSÃ_Ð-UÓVˆØ ×*Ñ*¨<×+<Ñ+<¸RÓ+@ÓAˆ	Ø˜ YÐ/Ð/r:   c                 ó¤  ‡ ‡— t        ‰ j                  «      dkD  r‰ j                  d   ||z  z  dk(  st        d‰ j                  › d|› �«      ‚d„ Šˆˆ fd„}‰ j                  d   ||z  z  }‰ j                  d   |z  }‰ j                  d	   |z  }g ‰ j                  d d ¢|‘|‘|‘­}‰ j                  |«      }|j	                   |«       ¬
«      }|S )Nrý   re  r   z'Invalid input shape for pixel_shuffle: z with upscale_factor = c                 ób   — t         j                  j                  | «      t         j                  k(  S rD   rÞ  rá  s    r8   rã  z,meta_pixel_shuffle.<locals>.is_channels_lastÛ  s$   € Ü×"Ñ"×8Ñ8¸Ó=Ä×ATÑATÑTÐTr:   c                  ó2  •—  ‰ ‰«      r.t        ‰«      dk(  rt        j                  S t        j                  S ‰j	                  t        j                  ¬«      rt        j                  S ‰j	                  t        j
                  ¬«      rt        j
                  S y )Nré   r«   )rð   rV   r°   rà  r–  Úpreserve_format)rã  rš   s   €€r8   rë  z.meta_pixel_shuffle.<locals>.pick_memory_formatÞ  s|   ø€ Ù˜DÔ!Ü˜4Ó  FÒ*Ü×.Ñ.Ð.ä×*Ñ*Ð*Ø×Ñ¬e×.EÑ.EÐÔFÜ×*Ñ*Ð*Ø×Ñ¬e×.CÑ.CÐÔDÜ×(Ñ(Ð(ð Er:   rò  rœ   r«   )rÃ   r™   rŠ   r˜   rY  )	rš   Úupscale_factorrë  rý  ÚHrÚWrr¨   rÍ   rã  s	   `       @r8   Úmeta_pixel_shufflerG	  Ò  så   ù€ ô 	ˆD�J‰J‹˜!Ò §
¡
¨2¡°.À>Ñ2QÑ RÐVWÒ WäØ5°d·j±j°\ÐAXÐYgÐXhÐió
ð 	
òUõ	)ð 	�
‰
�2‰˜>¨NÑ:Ñ;€AØ	�‰�B‰˜.Ñ	(€BØ	�‰�B‰˜.Ñ	(€BØ-�$—*‘*˜S˜b�/Ð- 1Ð- bÐ-¨"Ñ-€Ià
�.‰.˜Ó
#€CØ
�&‰&Ñ1Ó3ˆ&Ó
4€CØ€Jr:   c                 óX  — | j                  | j                  «      }|j                  |j                  «      }|j                  |j                  «      }|j                  |j                  «      }|j                  |j                  «      }|j                  |j                  «      }|||||||fS rD   r  )rØ   Úweight0Úweight1Úweight2Úweight3r	  Úcx_tmpr÷   Úhy_Úcy_Úgrad_output_r_optÚgrad_hy_r_optÚgrad_cy_r_optrÂ   rŒ  rë  rí  r	  rð  rñ  r*  rî  rã  Údiff_xÚdiff_hxÚdiff_cxÚdiff_w1Údiff_w2Údiff_bs                                r8   Úmkldnn_rnn_layer_backwardrY	  ó  s�   € ð4 �_‰_˜UŸ[™[Ó)€FØ�m‰m˜CŸI™IÓ&€GØ×Ñ˜vŸ|™|Ó,€GØ×Ñ §¡Ó.€GØ×Ñ §¡Ó.€GØ×Ñ˜wŸ}™}Ó-€FØ�7˜G V¨V°W¸gÐEÐEr:   )Ú	out_int32Úrightc                óŽ   — t        j                  | |rt         j                  nt         j                  t         j                  ¬«      S )N©r^   r¬   )rV   r¯   ru  r·   r°   ©rš   Ú
boundariesrZ	  r[	  s       r8   Úmeta_bucketizer`	    s2   € ô ×ÑØÙ&Œe�kŠk¬E¯K©KÜ×-Ñ-ôð r:   r_	  rZ	  r[	  c                ór   — |j                  d|rt        j                  ¬«      S t        j                  ¬«      S )Nr4   r‘   )r˜   rV   ru  r·   r^	  s       r8   Úmeta_bucketize_scalarrb	     s>   € ð ×ÑØ
Ù&Œe�k‰kð  ó ð ä,1¯K©Kð  ó ð r:   c                 óš  ‡ ‡‡‡‡— dŠt        ‰ «      dk(  r't        j                  ‰ j                  «       ˆ fd„«       t        ‰ «      dk(  r%‰ j                  «       rt	        j
                  d«       t        j                  t        ‰t        «      ˆˆfd„«       t        j                  ‰dkD  ˆˆfd„«       t        j                  t        ‰t        «      ˆˆfd	„«       t        j                  t        ‰t        «      ˆˆfd
„«       t        j                  ‰‰k\  ˆfd„«       t        j                  ‰‰ j                  ‰ j                  ¬«      S )Nzhistc()rè   c                  ó$   •— d‰ j                   › d�S )Nz%"histogram_cpu" not implemented for 'r  r‘   r  s   €r8   re   zmeta_histc.<locals>.<lambda>5  s   ø€ Ð=¸e¿k¹k¸]È!ÐL€ r:   ré   z%_histc_cuda with floating point inputc                  ó$   •— ‰› dt        ‰ «      › �S )Nz#: argument 'bins' must be int, not r/  ©Úbinsr  s   €€r8   re   zmeta_histc.<locals>.<lambda>;  s   ø€ �7�)Ð>¼tÀD»z¸lÐK€ r:   r   c                  ó   •— ‰› d‰ › �S )Nz: bins must be > 0, but got r4   rf	  s   €€r8   re   zmeta_histc.<locals>.<lambda>=  s   ø€  g YÐ.JÈ4È&Ð#Q€ r:   c                  ó$   •— ‰ › dt        ‰«      › �S )Nz%: argument 'min' must be Number, not r/  )r  rô   s   €€r8   re   zmeta_histc.<locals>.<lambda>@  ó   ø€ �7�)Ð@ÄÀcÃÀÐL€ r:   c                  ó$   •— ‰ › dt        ‰«      › �S )Nz%: argument 'max' must be Number, not r/  )r  rá  s   €€r8   re   zmeta_histc.<locals>.<lambda>D  rj	  r:   c                  ó   •— ‰ › d�S )Nz: max must be larger than minr4   )r  s   €r8   re   zmeta_histc.<locals>.<lambda>F  s   ø€ ¨ yÐ0MÐ%N€ r:   r¶   )rð   rV   rg   r  rO   r  rp   r   r   rŒ   rƒ   r^   )rØ   rg	  rô   rá  r  s   ````@r8   Ú
meta_histcrm	  .  sí   ü€ ð €GÜ�5Ó˜UÒ"Ü�‰Ø×#Ñ#Ó%ÛLô	
ô �5Ó˜VÒ#¨×(?Ñ(?Ô(AÜ×%Ñ%Ð&MÔNÜ	‡L�LÜ�4œÓ!ÜKôô 
‡L�L�˜‘ÔQÔRÜ	‡L�LÜ�3œÓÜLôô 
‡L�LÜ�3œÓÜLôô 
‡L�L�˜‘ÓNÔOÜ�;‰;�t E§L¡L¸¿¹ÔDÐDr:   c                 óB  ‡ — t        ‰ j                  «       |d¬«      }t        j                  ‰ j	                  «       dk7  xs# t        d„ ‰ j                  «       dd  D «       «      ˆ fd„«       ‰ j                  |«      j                  t        j                  ‰ «      ¬«      S )Nrý   r¬  r   c              3   ó&   K  — | ]	  }|d kD  –— Œ y­wr¢  r4   )rr   r£   s     r8   rt   z,meta_upsample_bimode2d_aa.<locals>.<genexpr>\  s   è ø€ Ò!H¨t $¨¥(Ñ!Hùr¤  r3   c                  ó*   •— d‰ j                  «       › �S r´  r¢   r  s   €r8   re   z+meta_upsample_bimode2d_aa.<locals>.<lambda>]  r«  r:   r«   )
r¨  r£   rV   rg   r—   rB  r˜   rY  rO   r   )rØ   rª  rø  r¶  r·  r¯  s   `     r8   Úmeta_upsample_bimode2d_aarq	  J  s‹   ø€ ô -Ø�
‰
‹�k°AôÐô 
‡L�LØ�‰‹˜ÑÒHœcÑ!H°u·z±z³|ÀAÀBÐ7GÔ!HÓHÛ_ôð �?‰?Ð+Ó,×/Ñ/Ü×1Ñ1°%Ó8ð 0ó ð r:   c                 óT  ‡ ‡‡— t        ||d¬«      Št        j                  ‰ j                  dk(  ˆ fd„«       t	        d«      D ]0  Št        j                  ‰ j
                  ‰   ‰‰   k(  ˆˆ ˆfd„«       Œ2 ‰ j                  |«      j                  t        j                  ‰ «      ¬«      S )Nrý   r¬  r5  c                  ó"   •— d‰ j                   › �S r¼  r7  rÐ  s   €r8   re   z4meta_upsample_bimode2d_aa_backward.<locals>.<lambda>w  r½  r:   c            
      óD   •— d‰› d‰ ‰   › d‰› d‰j                  ‰«      › �S )NzD
Expected grad_output to have the same shape as output; output.size(r¿  z
but got grad_output_size(r¢   rÀ  s   €€€r8   re   z4meta_upsample_bimode2d_aa_backward.<locals>.<lambda>|  s@   ø€ ð DØDEÀ3ÀdÐK[Ð\]ÑK^ÐJ_ð `Ø˜˜D ×!1Ñ!1°!Ó!4Ð 5ð9€ r:   r«   )
r¨  rV   rg   r¥   rÅ   r™   r˜   rY  rO   r   )r*  rª  r€  rø  r¶  r·  r¯  rÜ   s   `     @@r8   Ú"meta_upsample_bimode2d_aa_backwardru	  d  s¨   ú€ ô -Ø�K°!ôÐô 
‡L�LØ×Ñ˜AÑÛkôô �1‹Xò 
ˆÜ�‰Ø×Ñ˜aÑ Ð$4°QÑ$7Ñ7õ9õ	
ð
ð × Ñ  Ó,×/Ñ/Ü×1Ñ1°+Ó>ð 0ó ð r:   c                 óP  — t        j                  |j                  «       dk(  d„ «       t        j                  |j                  «       dk(  d„ «       t        j                  |j                  j                  d„ «       t        j                  |j                  j                  d„ «       y )Nr3   c                   ó   — y)Nz%found_inf must be a 1-element tensor.r4   r4   r:   r8   re   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>‰  rm   r:   c                   ó   — y)Nz%inv_scale must be a 1-element tensor.r4   r4   r:   r8   re   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>Œ  rm   r:   c                   ó   — y)Nz!found_inf must be a float tensor.r4   r4   r:   r8   re   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>�  rm   r:   c                   ó   — y)Nz!inv_scale must be a float tensor.r4   r4   r:   r8   re   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>”  rm   r:   )rV   rg   r—   r^   r  )rš   r+  Ú	inv_scales      r8   Ú*_amp_foreach_non_finite_check_and_unscale_r|	  †  s|   € ä	‡L�LØ�‰Ó˜QÑÑ Oôô 
‡L�LØ�‰Ó˜QÑÑ Oôô 
‡L�LØ�‰×)Ñ)Ù3ôô 
‡L�LØ�‰×)Ñ)Ù3õr:   c                 ó,   — t        j                  | «      S rD   rJ  )rš   ÚnanÚposinfÚneginfs       r8   Ú
nan_to_numr�	  ™  r¢  r:   c                 óî  — | j                   t        j                  t        j                  t        j                  t        j
                  hv rt        d| j                   › d�«      ‚| j                  }t        ||«      }t        ||«      }||k(  r| S t        | j                  «       «      }t        | j                  «       «      }||   ||   c||<   ||<   ||   ||   c||<   ||<   | j                  ||«       | S )Nz>torch.transpose_: in-place transposition is not supported for z layout)r‚   rV   rÙ  Ú
sparse_cscrÚ  Ú
sparse_bscrŠ   r¥   r¸   rÄ   r£   rÆ   rË   )rš   Údim0r’  Úndimsr£   rÆ   s         r8   rÍ  rÍ  Ÿ  só   € à‡{�{Ü×ÑÜ×ÑÜ×ÑÜ×Ñð	ñ ô ØLÈTÏ[É[ÈMÐY`Ðaó
ð 	
ð �I‰I€Eä˜$ Ó&€DÜ˜$ Ó&€Dàˆt‚|Øˆä�—	‘	“Ó€DÜ�$—+‘+“-Ó €Fà!'¨¡¨v°d©|Ð€Fˆ4�L�&˜‘,Ø! $™Z¨¨d©Ð€Dˆ�J��T‘
à×Ñ�T˜6Ô"Ø€Kr:   c                 ó  — | j                   }| j                  r<| j                  «       }| j                  «       }|dk  r|dk(  s4t	        d|› d|› d�«      ‚| j                  «       dkD  rt	        d|› d�«      ‚t        | d|dk  rd«      S d«      S )	Nrý   r   zEt_ expects a tensor with <= 2 sparse and 0 dense dimensions, but got z sparse and z dense dimensionsz6t_ expects a tensor with <= 2 dimensions, but self is r  r3   )r¥   r
  r  r  rŠ   r†   rÍ  )rš   r†	  r  r  s       r8   Út_rˆ	  ½  s¤   € à�I‰I€Eà‡~‚~Ø—_‘_Ó&ˆ
Ø—N‘NÓ$ˆ	Ø˜a’ I°¢NÜ ðØ%˜, l°9°+Ð=NðPóð ð
 �8‰8‹:˜Š>Ü ØHÈÈÈqÐQóð ô �d˜A E¨A¢I˜qÓ5Ð5°1Ó5Ð5r:   )rZ	  r[	  ÚsideÚsorterc                ó^  ‡ ‡‡— t        j                  t        ‰ j                  «      dk  xs ‰ j                  d d ‰j                  d d k(  ˆˆ fd„«       t        j                  ‰d u xs ‰ j                  ‰j                  k(  ˆ ˆfd„«       t        j                  |dk7  xs | d„ «       |rt         j                  nt         j
                  }t        ‰t         j                  «      r&t        j                  ‰|t         j                  ¬«      S t        j                  d|‰ j                  ¬	«      S )
Nr3   rœ   c                  ó`   •— dt        ‰j                  «      › dt        ‰ j                  «      › �S )Nz¯torch.searchsorted(): boundaries tensor should be 1 dimension or the first N-1 dimensions of boundaries tensor and input value tensor must match, but we got boundaries tensor z and input value tensor r@  )rš   Úsorted_sequences   €€r8   re   z#meta_searchsorted.<locals>.<lambda>â  s8   ø€ ð3ä37¸×8MÑ8MÓ3NÐ2Oð P"Ü"& t§z¡zÓ"2Ð!3ð5ð r:   c                  ól   •— dt        ‰ j                  «      › d‰�t        ‰j                  «      › �S g › �S )Nz[torch.searchsorted(): boundary and sorter must have the same size, but got boundary tensor z and got sorter tensor r@  )r�	  rŠ	  s   €€r8   re   z#meta_searchsorted.<locals>.<lambda>í  sO   ø€ ð#Ü#'¨×(=Ñ(=Ó#>Ð"?Ð?VØ%+Ð%7Œt�F—L‘LÓ!Ð@ðBð ð >@Ð@ðBð r:   rÕ   c                   ó   — y)Nzetorch.searchsorted(): side and right can't be set to opposites, got side of left while right was Truer4   r4   r:   r8   re   z#meta_searchsorted.<locals>.<lambda>÷  rm   r:   r]	  r4   ré  )rV   rg   rÃ   r™   ru  r·   rp   r   r¯   r°   rŒ   rƒ   )r�	  rš   rZ	  r[	  r‰	  rŠ	  r^   s   ``   ` r8   Úmeta_searchsortedr�	  Ò  sø   ú€ ô 
‡L�LÜˆO×!Ñ!Ó" aÑ'ò 	9Ø× Ñ   "Ð%¨¯©°C°R¨Ñ8ô	
ô	ô 
‡L�LØ�$ˆÒ?˜/×/Ñ/°6·<±<Ñ?ô	
ôô 
‡L�LØ�‰Ò#˜e˜)ñ	$ôñ %ŒE�KŠK¬%¯+©+€EÜ�$œŸ™Ô%Ü×ÑØ˜¬U×-DÑ-Dô
ð 	
ô �{‰{˜2 U°?×3IÑ3IÔJÐJr:   c                 ó–   ‡ — t        j                  ‰ t         j                  t         j                  t         j                  fvˆ fd„«       y )Nc                  ó   •— d‰ › �S )Nz/Unsupported input type encountered for isin(): r4   r‘   s   €r8   re   z3_check_for_unsupported_isin_dtype.<locals>.<lambda>   s   ø€ ÐAÀ%ÀÐI€ r:   )rV   rg   r   Ú
complex128Ú	complex64r‘   s   `r8   Ú!_check_for_unsupported_isin_dtyper•	     s/   ø€ Ü	‡L�LØ”e—j‘j¤%×"2Ñ"2´E·O±OÐDÐDÛIõr:   c                 óJ   — | j                  || j                  d«      f«      }|S )Nrœ   r'  )r*  r´   Únum_weightsrì  r÷  r>	  s         r8   Úmeta_embedding_dense_backwardr˜	     s*   € ð ×'Ñ'¨°k×6FÑ6FÀrÓ6JÐ(KÓL€KØÐr:   c                 ój   — |	rt         j                  | ||||||||
|«
      S t        | ||||||||
|«
      S rD   )r-   Ú_embedding_bag_sparse_backwardÚ!meta_embedding_bag_dense_backward)rx  r´   r©  rû  rü  Úmaximum_indicesr—	  r÷  rŒ  rø  ræ  rì  s               r8   Úmeta_embedding_bag_backwardr�	     se   € ñ Ü×2Ñ2ØØØØØØØØØØó
ð 	
ô 1ØØØØØØØØØØó
ð 	
r:   c
                 óN  ‡ — t        j                  ‰ j                  t         j                  t         j                  t         j
                  t         j                  fv ˆ fd„«       |t        k(  rt        j                  |d u«       ‰ j                  |‰ j                  d«      f«      }
|
S )Nc                  ó"   •— d‰ j                   › �S )Nz$Unsupported input type encountered: r‘   )rx  s   €r8   re   z3meta_embedding_bag_dense_backward.<locals>.<lambda>Q   s   ø€ Ð6°t·z±z°lÐC€ r:   r3   )
rV   rg   r^   r“  r”  r’  Úfloat64rõ  r˜   r£   )rx  r´   rû  rü  rœ	  r—	  r÷  rŒ  ræ  rì  Úindex_grad_weights   `          r8   r›	  r›	  B   sv   ø€ ô 
‡L�LØ�
‰
”u—}‘}¤e§n¡n´e·m±mÄUÇ]Á]ÐSÐSÛCôð ŒxÒÜ�‰�_¨DÐ0Ô1ØŸ™¨°T·Y±Y¸q³\Ð'BÓCÐØÐr:   c                 óÜ  — | j                  d«      }t        j                  |t        k(  d„ «       t        j                  | j	                  «       dk(  «       t        j                  |j	                  «       dk(  «       |j                  d«      }t        j                  |j	                  «       dk(  «       t        j                  |j                  d«      |k(  «       | j                  |f«      }	|	S )Nr3   c                   ó   — y)NzHembedding_bag_backward: per_sample_weights only supported for mode='sum'r4   r4   r:   r8   re   z@meta_embedding_bag_per_sample_weights_backward.<locals>.<lambda>f   rm   r:   rý   r   )r£   rV   rg   rô  r†   r˜   )
rx  rk  r´   r©  rû  rŒ  rì  Úembedding_featuresr”  r÷   s
             r8   Ú.meta_embedding_bag_per_sample_weights_backwardr¥	  Y   s«   € ð Ÿ™ 1›ÐÜ	‡L�LØ”ÑÙZôô 
‡L�L�—‘“˜q‘Ô!Ü	‡L�L�—‘“ !Ñ#Ô$Ø—,‘,˜q“/€KÜ	‡L�L�—‘“ Ñ"Ô#Ü	‡L�L�—‘˜Q“Ð#5Ñ5Ô6Ø�^‰^˜[˜NÓ+€FØ€Mr:   )Úassume_uniqueÚinvertc                óÒ  — t        j                  t        | t        «      xs t        |t        «      d„ «       t        | t        «      s!t        j                  | |j
                  ¬«      } t        |t        «      s!t        j                  || j
                  ¬«      }t        | j                  «       t        |j                  «       t        j                  | t         j                  ¬«      S )Nc                   ó   — y)Nz<At least one of elements and test_elements must be a Tensor.r4   r4   r:   r8   re   zmeta_isin.<locals>.<lambda>v   rm   r:   r7  r‘   )
rV   rg   rp   r   rb  rƒ   r•	  r^   r¯   r   )ÚelementsÚtest_elementsr¦	  r§	  s       r8   Ú	meta_isinr¬	  q   sš   € ô 
‡L�LÜ�8œVÓ$ÒI¬
°=Ä&Ó(IÙNôô �h¤Ô'Ü—<‘< °×1EÑ1EÔFˆä�m¤VÔ,ÜŸ™ ]¸8¿?¹?ÔKˆä% h§n¡nÔ5Ü% m×&9Ñ&9Ô:Ü×Ñ˜H¬E¯J©JÔ7Ð7r:   r  c                 ó    — t        j                  | dk\  d„ «       t        |t        j                  ¬«      \  }}t        j
                  ||¬«      S )Nr   c                   ó   — y)Nz,polygamma(n, x) does not support negative n.r4   r4   r:   r8   re   z meta_polygamma.<locals>.<lambda>†   rm   r:   rÈ  r‘   )rV   rg   r   r   rÉ  r¯   )r  rš   rR   rS   s       r8   Úmeta_polygammar¯	  ƒ   sF   € ô 
‡L�L��a‘ÑOÔPÜ(ØÜ;×HÑHô�O€A€|ô ×Ñ˜D¨Ô5Ð5r:   c                 ó   — t        d«      ‚)Nz.Tensor.item() cannot be called on meta tensors)rX  r±   s    r8   Úmeta_local_scalar_denser±	  Ž   s   € ä
ÐGÓ
HÐHr:   c                 ó,   — t        j                  | «      S rD   rJ  r±   s    r8   Úsilur³	  “   r¢  r:   c                 ól   — t        | t        j                  ¬«      \  }}t        j                  | |¬«      S rÇ  )r   r   rÉ  rV   r¯   )rš   rR   rS   s      r8   Úsigmoidrµ	  ™   s3   € ô )ØÜ;×HÑHô�O€A€|ô ×Ñ˜D¨Ô5Ð5r:   c                 óR  — | j                  «       dk(  }|j                  «       dk(  }|r‘|r4|j                  d«      | j                  d«      |j                  d«      g}�n"t        j                  |j                  d«      |j                  d«      k(  d„ «       | j                  d«      |j                  d«      g}nÇ|r[t        j                  |j                  d«      | j                  d«      k(  d„ «       | j                  d«      |j                  d«      g}njt        j                  | j                  d«      |j                  d«      k(  d„ «       | j                  d«      | j                  d«      |j                  d«      g}|xs | j                  }t        j
                  j                  rZd|j                  z  }|d   |z   dz
  |z  |z  }||k(  r|d   |z  |dg}	n|dg}	t        j                  ||	|| j                  ¬	«      }
|
S t        j                  ||| j                  ¬	«      }
|
S )
Nrý   r   r3   c                   ó   — y©Nz matrix batch sizes have to matchr4   r4   r:   r8   re   z2_create_grouped_mm_output_tensor.<locals>.<lambda>¬   rm   r:   rœ   c                   ó   — yr¸	  r4   r4   r:   r8   re   z2_create_grouped_mm_output_tensor.<locals>.<lambda>²   rm   r:   c                   ó   — y)Nzbatched dimension has to matchr4   r4   r:   r8   re   z2_create_grouped_mm_output_tensor.<locals>.<lambda>¸   rm   r:   rT  ré  )r†   r£   rV   rg   r^   rÖ  ré   ÚitemsizerX  rƒ   rŒ   )rz  r|  Úoffsro  Ú
mat1_is_2dÚ
mat2_is_2drø  Ú	alignmentÚsize_paddedrÉ  rÍ   s              r8   Ú _create_grouped_mm_output_tensorrÁ	  £   sÒ  € Ø—‘“˜q‘€JØ—‘“˜q‘€JáÙØŸ	™	 !› d§i¡i°£l°D·I±I¸a³LÐAŠHä�L‰LØ—	‘	˜!“ §	¡	¨!£Ñ,Ñ.Xôð Ÿ	™	 !› d§i¡i°£mÐ4‰HáÜ�L‰LØ—	‘	˜!“ §	¡	¨!£Ñ,Ñ.Xôð Ÿ	™	 !› d§i¡i°£lÐ3‰Hô �L‰LØ—	‘	˜!“ §	¡	¨!£Ñ,Ñ.Vôð Ÿ	™	 !› d§i¡i°£l°D·I±I¸b³MÐBˆHàÒ'˜TŸZ™Z€Iä‡}�}×ÒØ˜)×,Ñ,Ñ,ˆ	Ø ‘| iÑ/°!Ñ3¸	ÑAÀIÑMˆØ˜Ò#Ø" 1™+¨Ñ3°[À!ÐD‰Jà% qÐ)ˆJÜ×!Ñ!Ø�j¨	¸$¿+¹+ô
ˆð
 €Jô �k‰k˜(¨)¸D¿K¹KÔHˆØ€Jr:   Úmat_aÚmat_br¼	  c	                 óš  ‡ ‡‡‡‡‡— t        j                  ‰d u ‰d u k(  d„ «       ‰d uxr ‰d u}	|	r¼t         j                  }
t         j                  j                  rYt         j
                  j                  «       r;dt         j
                  j                  d«      j                  v rt         j                  }
t        j                  ‰ j                  |
k(  xr ‰j                  |
k(  ˆ ˆfd„«       nTt        j                  ‰ j                  t         j                  k(  xr ‰j                  t         j                  k(  ˆ ˆfd„«       t        j                  ‰ j                  «       dv xr ‰j                  «       dv ˆ ˆfd„«       ‰ j                  «       dk(  }‰j                  «       dk(  }|r|s8t        j                  ‰ j                  d	«      ‰j                  d
«      k(  d„ «       |	rDd„ }d„ }t        j                   |‰ «      ˆ fd„«       t        j                   |‰«      ˆfd„«       d„ } |d‰ «        |d‰«       ‰��‰��t        j                  ‰j                  t         j                  k(  xr ‰j                  t         j                  k(  xs< ‰j                  t         j                  k(  xr ‰j                  t         j                  k(  ˆˆfd„«       ‰j                  t         j                  k(  xr ‰j                  t         j                  k(  Šdˆfd„	}‰�|r|r‰j                   d   nd} |d‰‰ d|«        |d‰‰d|«       t        j                  |d u d„ «       |s|r}t        j                  ‰d uˆ ˆfd„«       ‰�xt        j                  ‰j                  «       dk(  ˆfd„«       t        j                  ‰j                  t         j"                  k(  ˆfd„«       nt        j                  ‰d u d„ «       t        j                  |d u d„ «       t        j                  |d u xs |t         j                  k(  d„ «       t%        ‰ ‰‰|«      S ) Nc                   ó   — y)Nz,Either both scale factors are given, or noner4   r4   r:   r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>Ú   rm   r:   Úgfx94r   c                  ó>   •— d‰ j                   › d‰j                   › d�S )Nz5Expected inputs of E4M3 FP8 type but got mat_a.dtype=ú and mat_b.dtype=rÂ  r‘   ©rÂ	  rÃ	  s   €€r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>ì   s%   ø€ ÐKÈEÏKÉKÈ=ÐXiÐjo×juÑjuÐivÐvwÐx€ r:   c                  ó>   •— d‰ j                   › d‰j                   › d�S )Nz1Expected inputs of BF16 type but got mat_a.dtype=rÈ	  rÂ  r‘   rÉ	  s   €€r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>ñ   s%   ø€ ÐGÈÏÉÀ}ÐTeÐfk×fqÑfqÐerÐrsÐt€ r:   )rý   r0   c                  óL   •— d‰ j                  «       › d‰j                  «       › �S )Nz3Multiplicands must be 2D or 3D but got mat_a.dim()=z and mat_b.dim()=r�   rÉ	  s   €€r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>ö   s'   ø€ ÐEÀeÇiÁiÃkÀ]ÐRcÐdi×dmÑdmÓdoÐcpÐq€ r:   rý   rœ   rò  c                   ó   — y)Nz3contraction dimension of mat_a and mat_b must matchr4   r4   r:   r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>ÿ   rm   r:   c                 óF   — | j                  «       }|d   dkD  xr |d   dk(  S ©Nrò  r3   rœ   rë  ©ÚmatÚ
mat_strides     r8   r  z-_meta_grouped_mm_common.<locals>.is_row_major!  s*   € ØŸ™›ˆJØ˜b‘> AÑ%Ò=¨*°R©.¸AÑ*=Ð=r:   c                 óF   — | j                  «       }|d   dk(  xr |d   dkD  S rÎ	  rë  rÏ	  s     r8   r
  z-_meta_grouped_mm_common.<locals>.is_col_major!  s*   € ØŸ™›ˆJØ˜b‘> QÑ&Ò=¨:°b©>¸AÑ+=Ð=r:   c                  ó0   •— d‰ j                  «       dd  › �S )NzNExpected mat_a tensor to be row major in the last two dimensions, got strides rò  rë  )rÂ	  s   €r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>!  s#   ø€ ÐdÐej×eqÑeqÓesÐtvÐtwÐexÐdyÐz€ r:   c                  ó0   •— d‰ j                  «       dd  › �S )NzQExpected mat_b tensor to be column major in the last two dimensions, got strides rò  rë  )rÃ	  s   €r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>!  s#   ø€ ÐgÐhm×htÑhtÓhvÐwyÐwzÐh{Ðg|Ð}€ r:   c                 óô  ‡ ‡‡‡— ‰j                  «       dz
  Šd‰j                  «       z  }‰j                  «       Š‰‰dz
     dk(  rG‰‰   t        d‰j                  ‰dz
     «      k\  r%t        j                  ‰‰   |z  dk(  ˆˆ ˆfd„«       y ‰‰   dk(  rJ‰‰dz
     t        d‰j                  ‰   «      k\  r(t        j                  ‰‰dz
     |z  dk(  ˆˆ ˆfd„«       y t        j                  dˆˆfd„«       y )Nr3   rT  r   c                  ó"   •— d‰› d‰ › d‰‰    › d�S )Nrû  ú stride along ú% dim to be multiple of 16 bytes, got rÂ  r4   ©Úend_dimÚmat_namerÑ	  s   €€€r8   re   zF_meta_grouped_mm_common.<locals>.check_valid_strides.<locals>.<lambda>!  s)   ø€ ˜) H :¨^¸G¸9ÐDiÐjtÐu|Ñj}Ði~Ð~ð  A€ r:   c                  ó.   •— d‰› d‰ dz
  › d‰‰ dz
     › d�S )Nrû  r×	  r3   rØ	  rÂ  r4   rÙ	  s   €€€r8   re   zF_meta_grouped_mm_common.<locals>.check_valid_strides.<locals>.<lambda>%!  sK   ø€ ˜) H :¨^¸GÀa¹K¸=ÐHmÐnxð  zAð  DEñ  zEñ  oFð  nGð  GHð  I€ r:   Fc                  ó*   •— d‰› d‰ j                   › d�S )NzInvalid strides/sizes, got z for strides and z for sizes.r$  rÏ	  s   €€r8   re   zF_meta_grouped_mm_common.<locals>.check_valid_strides.<locals>.<lambda>*!  s!   ø€ Ð5°j°\ÐARÐSV×S\ÑS\ÐR]Ð]hÐi€ r:   )r†   Úelement_sizerÆ   rá  r™   rV   rg   )rÛ	  rÐ	  r¿	  rÚ	  rÑ	  s   `` @@r8   Úcheck_valid_stridesz4_meta_grouped_mm_common.<locals>.check_valid_strides!  s   û€ Ø—'‘'“)˜a‘-ˆØ˜#×*Ñ*Ó,Ñ,ˆ	Ø—Z‘Z“\ˆ
Ø�g ‘kÑ" aÒ'¨J°wÑ,?Ä3Øˆs�y‰y˜ 1™Ñ%óD
ò -
ô �L‰LØ˜7Ñ# iÑ/°1Ñ4õ Aõð ˜Ñ  AÒ%¨*°W¸q±[Ñ*AÄSØˆs�y‰y˜Ñ!óF
ò +
ô �L‰LØ˜7 Q™;Ñ'¨)Ñ3°qÑ8õ Iõô
 �L‰LØÜiõr:   rÂ	  rÃ	  c                  ó>   •— d‰ j                   › d‰j                   › d�S )NzhFor FP8 scales must both be float32, or for MXFP8 both scales must be float8_e8m0fnu. Got scale_a.dtype=z and scale_b.dtype=rÂ  r‘   r&  s   €€r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>7!  sT   ø€ Ð~ð  @G÷  @Mñ  @Mð  Nð  Nað  bi÷  boñ  boð  apð  pqð  r€ r:   r3   c                 ó¼  •‡ ‡‡‡‡‡‡	‡
— ‰j                  «       dk(  rÍt        j                  ‰j                  «       ˆ fd„«       ‰r;t        j                  ‰j                  «       ‰j                  «       k(  ˆˆˆ fd„«       y t        j                  ‰j                  «       dk(  ˆˆ fd„«       t        j                  ‰j                  d   ‰j                  ‰   ‰z  k(  ˆˆˆˆ ˆfd„«       y t        j                  ‰j                  d«      dk(  ˆ fd	„«       t        j                  ‰j                  d   ‰j                  d   k(  ˆˆˆ fd
„«       ‰r§t        j                  ‰j                  ‰j                  dz
  k(  ˆˆˆ fd„«       ‰j                  \  Š}}d}t        ||z  d«      Š	t        |d«      Š
t        j                  ‰j                  d   ‰k(  xr ‰j                  d   ‰	‰
z  k(  ˆˆ	ˆ
ˆˆfd„«       y t        j                  ‰j                  «       dk(  ˆˆ fd„«       t        j                  ‰j                  d   ‰j                  d‰z      k(  ˆˆˆ ˆfd„«       y )Nrý   c                  ó   •— d‰ › d�S )Nrû  z to be contiguous.r4   ©Ú
scale_names   €r8   re   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>B!  s   ø€ ˜i¨
 |Ð3EÐF€ r:   c                  óB   •— d‰› d‰ j                   › d‰j                   › �S )NzKFor MXFP8, scale must have same number of dimensions as target tensor, but ú has mat.ndim=ú and scale.ndim=r7  ©rÐ	  r©  rä	  s   €€€r8   re   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>K!  sZ   ø€ Ð"mÐnxÐmyð  zHð  IL÷  IQñ  IQð  HRð  Rbð  ch÷  cmñ  cmð  bnð  !o€ r:   r3   c                  ó2   •— d‰› d‰ j                  «       › d�S )Nrû  z to be 1D tensor, but got ú	D tensor.r�   ©r©  rä	  s   €€r8   re   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>P!  ó#   ø€  )¨J¨<Ð7QÐRW×R[ÑR[ÓR]ÐQ^Ð^gÐ h€ r:   r   c                  óV   •— d‰› d‰ j                   ‰   ‰z  › d‰j                   d   › d�S )Nrû  z	 to have r!  r   z
 elements.r$  )rÐ	  r©  Úscale_multiplierrä	  Ú
scaled_dims   €€€€€r8   re   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>T!  sW   ø€  )¨J¨<°yÀÇÁÈ:ÑAVÐYiÑAiÐ@jÐjyÐz÷  {Fñ  {Fð  GHñ  {Ið  zJð  JTð  !U€ r:   rœ   c                  ó   •— d‰ › d�S )Nrû  z( to be contiguous in the last dimension.r4   rã	  s   €r8   re   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>Y!  s   ø€ ˜i¨
 |Ð3[Ð\€ r:   c                  óP   •— d‰› d‰ j                   d   › d‰j                   d   › d�S )Nrû  z batch dimension to be r   ú, got rÂ  r$  rè	  s   €€€r8   re   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>]!  s6   ø€ ˜i¨
 |Ð3JÈ3Ï9É9ÐUVÉ<È.ÐX^Ð_d×_jÑ_jÐklÑ_mÐ^nÐnoÐp€ r:   c                  óB   •— d‰› d‰ j                   › d‰j                   › �S )Nz0For MXFP8, 3d tensor should have 2d scales, but ræ	  rç	  r7  rè	  s   €€€r8   re   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>d!  sF   ø€ Ð"RÐS]ÐR^Ð^lÐmp×muÑmuÐlvð  wGð  HM÷  HRñ  HRð  GSð  !T€ r:   rU  r5  ro  c                  óN   •— d‰j                   › d‰ › d‰‰z  › d‰j                   › �S )NzFor MXFP8, expected mat.shape=z to have scale shape of (ú,z), but got r$  )ÚGÚ	blocked_KÚ	blocked_NrÐ	  r©  s   €€€€€r8   re   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>n!  sQ   ø€ Ð"@ÀÇÁÀÐKdÐefÐdgÐghÐirÐu~Ñi~Ðhð  @Kð  LQ÷  LWñ  LWð  KXð  !Y€ r:   c                  ó2   •— d‰› d‰ j                  «       › d�S )Nrû  z to be 2D tensor, but got rê	  r�   rë	  s   €€r8   re   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>s!  rì	  r:   c                  óV   •— d‰› d‰ j                   d‰z      › d‰j                   d   › d�S )Nrû  z non-batch dimension to be r3   rò	  rÂ  r$  )rÐ	  r©  rä	  rï	  s   €€€€r8   re   z>_meta_grouped_mm_common.<locals>.check_scale.<locals>.<lambda>w!  sT   ø€  )¨J¨<Ð7RÐSV×S\ÑS\Ð]^ÐakÑ]kÑSlÐRmÐmsÐty×tÑtð  ABñ  uCð  tDð  DEð  !F€ r:   )r†   rV   rg   r–  r™   rÆ   r¥   r?   )rä	  r©  rÐ	  rï	  rî	  rŸ  r¹  rª  rö	  r÷	  rø	  Úis_mxfp8s   `````   @@@€r8   Úcheck_scalez,_meta_grouped_mm_common.<locals>.check_scale>!  sÚ  ÿø€ Ø�w‰w‹y˜AŠ~Ü—‘Ø×'Ñ'Ó)ÛFôñ Ü—L‘LØŸ	™	› s§w¡w£yÑ0õ oõô
 —L‘LØŸ	™	› qÑ(Ühôô —L‘LØŸ™ A™¨#¯)©)°JÑ*?ÐBRÑ*RÑR÷ Uõô
 —‘Ø—L‘L Ó$¨Ñ)Û\ôô —‘Ø—K‘K ‘N c§i¡i°¡lÑ2Ýpôñ Ü—L‘LØŸ
™
 c§h¡h°¡lÑ2õ Tôð "Ÿi™i‘G�A�q˜!Ø!#�JÜ (¨¨Z©¸Ó ;�IÜ (¨¨CÓ 0�IÜ—L‘LØŸ™ A™¨!Ñ+ÒW°·±¸A±À)ÈiÑBWÑ0W÷ Yõô
 —L‘LØŸ	™	› qÑ(Ühôô —L‘LØŸ™ A™¨#¯)©)°A¸
±NÑ*CÑCö Fõr:   rò  ró  c                   ó   — y)Nz:Scale result tensor provided, but it is not supported yet.r4   r4   r:   r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>‚!  rm   r:   c                  óN   •— d‰ j                  «       › d‰j                  «       › d�S )Nz/Offsets tensor not provided, but is needed for zD/zD multiplicand layouts.r�   rÉ	  s   €€r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>ˆ!  s*   ø€ ÐEÀeÇiÁiÃkÀ]ÐRTÐUZ×U^ÑU^ÓU`ÐTaÐaxÐy€ r:   c                  ó,   •— d‰ j                  «       › d�S )Nz.Offsets tensor must be 1D, but got offs.dim()=rÂ  r�   ©r¼	  s   €r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>�!  s   ø€ ÐHÈÏÉËÈÐTUÐV€ r:   c                  ó$   •— d‰ j                   › d�S )Nz7Offsets tensor must be integer (int32) tensor, but got rÂ  r‘   r 
  s   €r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>‘!  s   ø€ ÐQÐRV×R\ÑR\ÐQ]Ð]^Ð_€ r:   c                   ó   — y)NzJOffsets tensor provided, but is not needed for 3D/3D multiplicand layouts.r4   r4   r:   r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>–!  rm   r:   c                   ó   — y)Nz2Bias tensor provided, but it is not supported yet.r4   r4   r:   r8   re   z)_meta_grouped_mm_common.<locals>.<lambda>›!  rm   r:   c                   ó   — y)Nz4If output dtype provided, it must be torch.bfloat16.r4   r4   r:   r8   re   z)_meta_grouped_mm_common.<locals>.<lambda> !  rm   r:   r/  )rV   rg   r•  rÖ  r×  ré   ró   Úget_device_propertiesÚgcnArchNamerú  r^   r”  r†   r£   r’  r3  r™   ru  rÁ	  )rÂ	  rÃ	  rò  ró  r¼	  rm  rô  ro  rõ  ÚscaledÚ	fp8_dtypeÚmat_a_is_2dÚmat_b_is_2dr  r
  rß	  rü	  rî	  rû	  s   `````             @r8   Ú_meta_grouped_mm_commonr
  Í   s�  ý€ ô 
‡L�LØ	�Dˆ˜g¨˜oÑ.Ù>ôð ˜DÐ Ò8 W°DÐ%8€Fñ Ü×'Ñ'ˆ	ä�M‰M×ÒÜ—
‘
×'Ñ'Ô)Øœ5Ÿ:™:×;Ñ;¸AÓ>×JÑJÑJä×-Ñ-ˆIÜ�‰Ø�K‰K˜9Ñ$ÒA¨¯©¸	Ñ)AÜxõ	
ô
 	�‰Ø�K‰Kœ5Ÿ>™>Ñ)ÒK¨e¯k©k¼U¿^¹^Ñ.KÜtô	
ô
 
‡L�LØ�	‰	‹�vÐÒ7 %§)¡)£+°Ð"7Üqôð
 —)‘)“+ Ñ"€KØ—)‘)“+ Ñ"€Ká™kÜ�‰Ø�J‰J�r‹N˜eŸj™j¨›nÑ,ÙIô	
ñ
 ò	>ò	>ô 	�‰Ù˜ÓÛzô	
ô 	�‰Ù˜ÓÛ}ô	
ò
ñ0 ˜ Ô'Ù˜ Ô'àÑ˜wÑ2Ü�‰Ø�]‰]œeŸm™mÑ+ÒN°·±ÄÇÁÑ0Nò à—‘¤×!5Ñ!5Ñ5ò :Ø—M‘M¤U×%9Ñ%9Ñ9ô rô	
ð �M‰MœU×1Ñ1Ñ1ò 6Ø—‘¤×!5Ñ!5Ñ5ð 	õ
:	ðz "Ð-±+Á+ˆD�J‰J�qŠMÐSTð 	ñ 	�I˜w¨¨qÐ2BÔCÙ�I˜w¨¨qÐ2BÔCä�‰Ø˜DÐ ÙPô	
ñ
 ‘kÜ�‰Ø˜ÐÜyô	
ð ÐÜ�L‰LØ—‘“
˜a‘ÛVôô �L‰LØ—
‘
œeŸk™kÑ)Û_õô
 	�‰Ø�DˆLÙ`ô	
ô
 
‡L�LØ�ˆÙDôô
 
‡L�LØ�TÐÒ8˜Y¬%¯.©.Ñ8ÙFôô
 ,¨E°5¸$À	ÓJÐJr:   c           
      ó(   — t        | |d d ||d |¬«      S )N)rò  ró  r¼	  rm  rô  ro  )r
  )rÂ	  rÃ	  r¼	  rm  ro  s        r8   Úmeta_grouped_mmr
  ¦!  s)   € ô #ØØØØØØØØô	ð 	r:   c	                 óR   — |xs t         j                  }t        | ||||||||¬«	      S )N)rò  ró  r¼	  rm  rô  ro  rõ  )rV   r”  r
  )	rÂ	  rÃ	  rò  ró  r¼	  rm  rô  ro  rõ  s	            r8   Úmeta_scaled_grouped_mmr
  »!  s:   € ð Ò+œUŸ^™^€Iä"ØØØØØØØ!ØØ%ô
ð 
r:   c                 ó6  — t        |«      t        d«      k(  r/| D ]*  }t        j                  |j                  «       dkD  d„ «       Œ, g }| D ]K  }|�|n|j                  }|j
                  rt        |«      }|j                  |j                  d|¬«      «       ŒM |S )NÚinfr   c                   ó   — y)Nz:_foreach_norm cannot compute infinity norm on empty tensorr4   r4   r:   r8   re   z#meta_foreach_norm.<locals>.<lambda>Ý!  rm   r:   r4   r‘   )	rZ   rV   rg   r—   r^   rî   r   rÔ  r˜   )ÚtensorsÚordr^   r  Úresultsro  s         r8   Úmeta_foreach_normr
  ×!  s–   € äˆSƒz”U˜5“\Ò!Øò 	ˆAÜ�L‰LØ—‘“	˜A‘ÙTõð	ð
 €GØò 9ˆØ"Ð.‘E°A·G±Gˆ	Ø×ÒÜ0°Ó;ˆIØ�‰�q—{‘{ 2¨Y�{Ó7Õ8ð	9ð
 €Nr:   r=   Úhalf_to_floatc                 óP  — |rE| j                   t        j                  t        j                  fvrt	        d| j                   › d�«      ‚t        j                  | t
        j                  j                  ¬«      \  }}|s|n|}t        j                  | |t        j                  ¬«      }|S )Nz%half_to_float is True but x.dtype is z, expected half or bfloat16rÈ  r]	  )r^   rV   rX   r”  rŠ   rO   r   r   rP   r¯   r°   )r=   r†   r
  Úcomputation_dtyperS   rÇ  s         r8   Úsoftmaxr
  è!  s’   € ñ Ø�7‰7œ5Ÿ:™:¤u§~¡~Ð6Ñ6Ü Ø7¸¿¹°yÐ@[Ð\óð ô ',×&>Ñ&>Ø	œu×DÑD×LÑLô'Ñ#Ð�|ñ (5‘<Ð:K€LÜ
×
Ñ
˜1 LÄ×@WÑ@WÔ
X€CØ€Jr:   c           	      ó°  ‡‡‡‡	‡
‡— t        j                  t        ‰«      dz  dk(  ˆfd„«       | j                  Št        ‰«      Š
t        ‰«      dz  }‰
|z
  Š	t        j                  ‰
|k\  ˆ
ˆfd„«       t	        d„ ‰D «       «      r•| }t        ‰	‰
«      D ]t  Šd‰
‰z
  dz
  z  Š‰‰   dk  r*|j                  ‰‰‰    |j                  ‰   ‰‰   z   «      }‰‰dz      dk  sŒL|j                  ‰d|j                  ‰   ‰‰dz      z   «      }Œv |j                  «       S t        ‰d ‰	 «      }t        |«      D ]^  Št        ‰«      ‰dz   dz  z
  Š‰‰	‰z      ‰‰   z   ‰‰dz      z   }t        j                  |dk\  ˆˆˆ	ˆˆfd„«       |j                  |«       Œ` t        j                  || j                  | j                  | j                  t        | «      ¬«      S )	Nrý   r   c                  ó    •— dt        ‰ «      › �S )Nz1Length of pad must be even but instead it equals rø  r„  s   €r8   re   z'_constant_pad_nd_meta.<locals>.<lambda> "  s   ø€ ÐCÄCÈÃHÀ:ÐN€ r:   c                  ó(   •— dt        ‰«      › d‰ › d�S )Nz`Length of pad should be no more than twice the number of dimensions of the input. Pad length is z while the input has z dimensions.rø  )Úl_inpr…  s   €€r8   re   z'_constant_pad_nd_meta.<locals>.<lambda>
"  s"   ø€ ð 2Ü25°c³(°Ð;PØˆ'�ð€ r:   c              3   ó^   K  — | ]%  }t        |t        j                  «      xr |d k  –— Œ' y­wr¢  )rp   rO   ÚIntWithoutSymInt)rr   rt  s     r8   rt   z(_constant_pad_nd_meta.<locals>.<genexpr>"  s)   è ø€ Ò
IÀŒ:�aœ×/Ñ/Ó0Ò;°Q¸!±VÓ;Ñ
Iùs   ‚+-r3   c            	      óF   •— d‰‰‰ z      › d‰‰   › d‰‰dz      › d‰‰ z   › d�	S )NzThe input size z, plus negative padding r¡   r3   zG resulted in a negative output size, which is invalid. Check dimension z of your input.r4   )rÜ   r  Úl_diffr…  Úpad_idxs   €€€€€r8   re   z'_constant_pad_nd_meta.<locals>.<lambda>#"  sG   ø€ �o k°&¸1±*Ñ&=Ð%>Ð>VØ�7‰|ˆn˜E # g°¡kÑ"2Ð!3ð 41Ø17¸!±°¸OðM€ r:   )r^   rƒ   r…   r¬   )rV   rg   rÃ   r™   rB  rÅ   Únarrowrï   rÄ   rÔ  rŒ   r^   rƒ   r…   r   )rØ   r…  rV  Úl_padÚc_inputÚ	new_shapeÚnew_dimrÜ   r  r"
  r
  r#
  s    `     @@@@@r8   Ú_constant_pad_nd_metar)
  ú!  sç  ý€ ô 
‡L�LÜˆC‹�1‰˜ÑÛNôð
 —+‘+€KÜ�Ó€EÜ�‹H˜‰M€EØ�U‰]€Fä	‡L�LØ�‰ô	ôô Ñ
IÀSÔ
IÔIØˆÜ�v˜uÓ%ò 	TˆAØ˜5 1™9 q™=Ñ)ˆGØ�7‰|˜aÒØ!Ÿ.™.Ø˜˜G™�} g§m¡m°AÑ&6¸¸W¹Ñ&Eó�ð �7˜Q‘;Ñ !Ó#Ø!Ÿ.™.¨¨A¨w¯}©}¸QÑ/?À#ÀgÐPQÁkÑBRÑ/RÓS‘ð	Tð �}‰}‹Ðä�[  &Ð)Ó*€IÜ�5‹\ò 	"ˆÜ�c“(˜q 1™u¨™kÑ*ˆØ˜f q™jÑ)¨C°©LÑ8¸3¸wÈ¹{Ñ;KÑKˆÜ�‰Ø�q‰L÷Mô	
ð 	×Ñ˜Õ!ð	"ô �;‰;ØØ�k‰kØ�|‰|Ø×)Ñ)Ü+¨EÓ2ôð r:   rì  r÷  rø  c                 óD  — | j                  «       dk7  rt        d| j                  «       › d�«      ‚| j                  }|j                  }|j                  dk(  r|d   f}n$|j                  dk(  r|d   |d   f}n
g |¢|d   ‘­}| j                  }| j                  ||¬«      S )Nrý   z'weight' must be 2-D, got z-Dr   r3   r‘   )r†   rŠ   r™   r¥   r^   r˜   )	rk  r´   rì  r÷  rø  Úweight_shapeÚindices_shaper¨   ro  s	            r8   Ú	embeddingr-
  2"  s¨   € ð ‡z�zƒ|�qÒÜÐ9¸&¿*¹*»,¸ÀrÐJÓKÐKØ—<‘<€LØ—M‘M€Mà‡|�|�qÒØ&2°1¡oÐ%7‰	Ø	�‰˜Ò	Ø" 1Ñ% |°A¡Ð7‰	à5�mÐ5 \°!¡_Ñ5ˆ	à—‘€IØ×Ñ˜I¨YÐÓ7Ð7r:   Úmax_lengthsÚpadding_valuec                 ó  — t        |«      dk7  rt        dt        |«      › d�«      ‚t        |«      dk7  rt        dt        |«      › d�«      ‚|d   j                  d   dz
  }|d   }||g| j                  dd  ¢­}| j                  |«      S )Nr3   z&Only one jagged dim is supported, got z offsetsz max_lengthsr   )rÃ   rŠ   r™   r˜   )r³   r©  r.
  r/
  r  r§  r™  s          r8   Ú$meta__jagged_to_padded_dense_forwardr1
  K"  sª   € ô ˆ7ƒ|�qÒÜØ4´S¸³\°NÀ(ÐKó
ð 	
ô ˆ;Ó˜1ÒÜØ4´S¸Ó5EÐ4FÀlÐSó
ð 	
ð 	�‰
×Ñ˜Ñ˜aÑ€AØ�A‰€AØ�qÐ,˜6Ÿ<™<¨¨Ð+Ñ,€LØ×Ñ˜LÓ)Ð)r:   c                 óB   — t        | «      t        «       d„ «       «       }|S )Nc                 ó8   — t        | t        j                  ¬«      S r'  ©rT   r   rÉ  r<  s    r8   Ú_fz)_create_unary_float_meta_func.<locals>._fc"  s   € ô  ØÔ=×JÑJô
ð 	
r:   ©rK   r$   ©Úfuncr5
  s     r8   Ú_create_unary_float_meta_funcr9
  b"  ó*   € Ü�4ÓÜƒ]ñ
ó ó ð
ð
 €Ir:   c                 óú  — | j                   s|j                   s|j                   rt        d«      ‚| j                  «       dk(  r,| j                  | j                  «      | j                  d«      fS | j                  d«      }| j                  d«      }|j                  d«      }| j                  |||«      }|
r1|r| j                  |||«      }||fS | j                  ||||«      }||fS | j                  d«      }||fS )NzP_native_multi_head_attention fake implementation does not support nested tensorsr   r3   )Ú	is_nestedr²  r—   r˜   r™   r£   )ri  rÁ   rV  Ú	embed_dimr¶  Ú
qkv_weightÚqkv_biasÚproj_weightÚ	proj_biasr±  Úneed_weightsÚaverage_attn_weightsÚ	mask_typer  ÚTÚ
output_dimr÷   Úattn_weightss                     r8   Ú native_multi_head_attention_fakerH
  n"  sû   € ð  ‡‚˜#Ÿ-š-¨5¯?ª?Ü!Ø^ó
ð 	
ð ‡{�{ƒ}˜ÒØ—‘ §¡Ó,¨e¯o©o¸aÓ.@ÐAÐAà�
‰
�1‹€AØ�
‰
�1‹€Að ×!Ñ! !Ó$€JØ�_‰_˜Q  :Ó.€FáÙð !Ÿ?™?¨1¨a°Ó3ˆLð �LÐ!Ð!ð	 !Ÿ?™?¨1¨h¸¸1Ó=ˆLð �LÐ!Ð!ð —‘ qÓ)ˆà�LÐ!Ð!r:   c                 óB   — t        | «      t        «       d„ «       «       }|S )Nc                 ó:   — t        | |t        j                  ¬«      S r'  r4
  r<   s     r8   r5
  z*_create_binary_float_meta_func.<locals>._f "  s   € ô  ØˆqÔ!@×!MÑ!Mô
ð 	
r:   r6
  r7
  s     r8   Ú_create_binary_float_meta_funcrK
  Ÿ"  r:
  r:   c                 ó˜   ‡ — t        ‰ «      ˆ fd„«       }‰ j                  › d�}||_         t        t        t        |«      «      |«      }|S )Nc                 ó`   •—  ‰| g|¢­i |¤Ž}t        | j                  |j                  «       | S rD   rU  )rš   rQ   r(  rÍ   rF   s       €r8   Ú_fnz#_register_inplace_meta.<locals>._fnÄ"  s.   ø€ á�Ð'˜Ò' Ñ'ˆÜ §
¡
¨C¯I©IÔ6Øˆr:   rR   )r   r{   rK   Úgetattrr-   )rF   rN
  Úinplace_names   `  r8   Ú_register_inplace_metarQ
  Ã"  sO   ø€ Ü
ˆ2ƒYóó ðð
 —k‘k�] !Ð$€LØ€C„LØ
4Œ-œ¤ lÓ3Ó
4°SÓ
9€Cà€Jr:   c                 óf  ‡ ‡‡— t        j                  ‰ j                  ‰j                  k(  ˆˆ fd„«       ‰ ‰g}t        ‰t        «      rQ‰j
                  dk7  r1t        j                  ‰ j                  ‰j                  k(  ˆ ˆfd„«       |j                  ‰«       t        |dt        j                  iŽS )Nc                  ó<   •— d‰j                   › d‰ j                   › �S )Nr·  z for `end`, but got dtype r‘   )r|   r}   s   €€r8   re   zlerp.<locals>.<lambda>Ö"  s   ø€ �/ %§+¡+ Ð.HÈÏÉÈÐT€ r:   r   c                  ó<   •— d‰ j                   › d‰j                   › �S )Nr·  z for `weight`, but got dtype r‘   )r}   rk  s   €€r8   re   zlerp.<locals>.<lambda>Ý"  s!   ø€ ˜/¨%¯+©+¨Ð6SÐTZ×T`ÑT`ÐSaÐb€ r:   rL   )
rV   rg   r^   rp   r   r¥   rÔ  rT   r   rP   )r}   r|   rk  rQ   s   ``` r8   ÚlerprU
  Ñ"  s�   ú€ ô 
‡L�LØ�‰�s—y‘yÑ ÜTôð �3ˆ<€DÜ�&œ*Ô%Ø�;‰;˜!ÒÜ�L‰LØ—‘˜vŸ|™|Ñ+Übôð 	�‰�FÔÜØ	ðÜ=×EÑEñð r:   )rV  c                ó<   — t        | ||t        j                  ¬«      S r'  r(  ©rØ   Útensor1Útensor2rV  s       r8   ÚaddcmulrZ
  å"  s!   € ô Øˆw˜Ô0O×0WÑ0Wôð r:   c                óè   — t        j                  t        j                  |j                  «      xr t        j                  |j                  «       d„ «       t        | ||t        j                  ¬«      S )Nc                   ó   — y)N)zFInteger division with addcdiv is no longer supported, and in a future zErelease addcdiv will perform a true division of tensor1 and tensor2. z4The historic addcdiv behavior can be implemented as zA(input + value * torch.trunc(tensor1 / tensor2)).to(input.dtype) zfor integer inputs and as z6(input + value * tensor1 / tensor2) for float inputs. z?The future addcdiv behavior is just the latter implementation: z4(input + value * tensor1 / tensor2), for all dtypes.r4   r4   r:   r8   re   zaddcdiv.<locals>.<lambda>õ"  rm   r:   râ  )rV   rg   rO   rm  r^   rT   r   rP   rW
  s       r8   Úaddcdivr]
  í"  sb   € ô 
‡L�Lä×"Ñ" 7§=¡=Ó1ò 6Ü×&Ñ& w§}¡}Ó5ð	
ñ		
ôô  Øˆw˜Ô0O×0WÑ0Wôð r:   c                  óÚ  — i } dD ]  }t         |   }|D ]  }|| vsŒ||   | |<   Œ Œ! | j                  «       D �]±  \  }}t        |t        j                  j
                  «      rŒ,t        |t        «      st        dt        |«      › �«      ‚ |j                  t        j                  j                  j                  «      |«       t        j                  j                  |j                  «       d«      r|t         d   v sŒÆt        |› d�«      ‚|j                   rŒá|j                  «       dv rŒôd|j                  «       v rt"        j%                  ||«       �Œd|j                  «       v rt&        j%                  ||«       �ŒHd	|j                  «       v rt(        j%                  ||«       �Œrd
|j                  «       v rt*        j%                  ||«       �Œœt,        j%                  ||«       �Œ´ y )N)r€   Úpost_autogradÚpre_autogradz$op_overload must be OpOverload, got ÚCompositeImplicitAutogradr€   z« is a CompositeImplicitAutograd op, we shouldn't register meta function for it. Instead, we should let the decomposition run and write meta kernels for the base operators.>   úaten::cloneúaten::copy_úaten::rot90úaten::_to_copyúaten::empty_stridedúaten::constant_pad_ndúaten::as_strided_scatterzmkldnn::zmkl::zonednn::zquantized::)r   Úitemsrp   rV   Ú_opsÚHigherOrderOperatorr   rŠ   rz   Úpy_implÚ_CÚDispatchKeyr/   Ú%_dispatch_has_kernel_for_dispatch_keyr÷  rX  Úis_viewÚ2_meta_lib_dont_use_me_use_register_meta_for_mkldnnÚimplÚ/_meta_lib_dont_use_me_use_register_meta_for_mklÚ2_meta_lib_dont_use_me_use_register_meta_for_onednnÚ5_meta_lib_dont_use_me_use_register_meta_for_quantizedÚ'_meta_lib_dont_use_me_use_register_meta)Úactivate_meta_tableÚtypÚregistryÚopoÚop_overloadrF   s         r8   Úactivate_metar|
  #  sæ  € ØÐð 9ò 9ˆÜ-¨cÑ2ˆàò 	9ˆCØÐ-Ò-Ø+3°C©=Ð# CÒ(ñ	9ð9ð /×4Ñ4Ó6ó 9N‰ˆ�Rô
 �k¤5§:¡:×#AÑ#AÔBØÜ˜+¤zÔ2Ü Ø6´t¸KÓ7HÐ6IÐJóð ð 	7ˆ×ÑœEŸH™H×0Ñ0×5Ñ5Ó6°rÔ:ä�8‰8×9Ñ9Ø×ÑÓÐ ;ô
ð Ô8¸Ñ@Ò@Ü"Ø"�mð $;ð ;óð ð
 × Ò ð à×ÑÓðñ	ð à˜[×-Ñ-Ó/Ñ/ÜB×GÑGÈÐUWÖXØ˜K×,Ñ,Ó.Ñ.Ü?×DÑDÀ[ÐRTÖUØ˜{×/Ñ/Ó1Ñ1ÜB×GÑGÈÐUWÖXØ +×"2Ñ"2Ó"4Ñ4ÜE×JÑJØ öô 8×<Ñ<¸[È"ÖMñs9Nr:   )F)ç        g      ð?r7	  rD   )NNNFr   r3   rœ   rÒ  )Trþ  )r  )r  T)FF)TT)rŽ  )FTN)TFF)TF)rý   )g      à?N)r@   Ústr)r4   rì  r/  F)r4   rì  FTN)Fr   FNFrœ   )NF)rœ   F)g      À?gUUUUUUÕ?FN)NNNNN)r   NNr3   )NNF)r}
  FFN)Nr}
  FFN)r}
  FNN)Nr}
  FNNF)r}
  FN)FN)NNNNNNN)FNNNN)NNNF)NNNNF)Nrœ   FNN)NNNN)rœ   TT)NNr   N)éd   r   r   )rœ   )rý   N)rœ   FF)r}
  )NTTN(Á  rÕ  Úcollections.abcr   r   Úenumr   Ú	functoolsr   Útypingr   Útyping_extensionsr   rV   Útorch._prims_commonrß  rO   r	   r
   r   Útorch._decompr   r   r   r   Ú
torch._opsr   Útorch._primsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Útorch._prims_common.wrappersr    r!   r"   r#   r$   r  r%   r&   Útorch.fx.experimentalr'   r�  Útorch.nn.functionalr(   r)   Útorch.utilsr*   rH   r+   r,   Úopsr-   ÚlibraryÚLibraryrv
  rÅ   rô  rö  rõ  r9   r?   rK   rT   r`   ri   ÚlinspaceÚlogspacerØ  rŽ   Útaker  rÍ   r›   r©   r²   ÚcummaxÚcumminr¹   r¼   rß   rÄ   rã   r   ræ   Ú_fft_c2crû   rõ   rþ   Ú_fft_r2cr	  ÚrandpermÚgenerator_outr  r•   r  Úrandintr  r  Úlow_outr  Úrandr!  Ú_philox_key_splitr+  Ú_philox_key_fold_inr0  rD  rF  rJ  rL  rM  Ú_fft_c2rrR  rÍ  r_  rf  Ú
unsqueeze_rj  Ú_sparse_semi_structured_linearr~
  r^   ry  Ú_sparse_semi_structured_mmr†  Ú_sparse_semi_structured_addmmrŠ  Ú_cslt_sparse_mmrš  Úindex_reducer¡  Úindex_reduce_r£  Úindex_selectr§  Úsegment_reducer³  rá  Ú	unary_outr·  r†   rÀ  rô   rÃ  rÅ  rÎ  rÊ  rÏ  Ú_assert_asyncrÒ  ÚmsgrÖ  Ú_printrÙ  Ú_make_dep_tokenrÜ  rã  Ú_functional_sym_constrain_rangerè  rë  Ú(_functional_sym_constrain_range_for_sizerí  Ú_functional_assert_asyncrï  r­   r   r®   r
  r  r  r  Ú_linalg_eighr  r&  Ú_linalg_eigvalsÚlinalg_eigvalsr)  Ú
linalg_eigr-  r1  r3  r9  r>  rB  rH  rZ  Úlinalg_inv_exr]  Úlinalg_ldl_factor_exrf   rc  Úlinalg_ldl_solverq  Ú	linalg_lurx  Úlinalg_lu_factor_exr|  Úlinalg_lu_solver…  Ú	lu_unpackr‹  r”  Ú	linalg_qrr›  rŸ  rœ  Ú_linalg_svdr¨  rn  r:  r·  rÉ  Úlinalg_solve_triangularrÐ  rÖ  rß  Ú_linalg_detrâ  rõ  r   r  Úreflection_pad1dr  Úreplication_pad1dr!  r,  Úreflection_pad1d_backwardr1  Úreplication_pad1d_backwardr3  rC  Úreflection_pad2drE  Úreplication_pad2drH  Ú_weight_norm_interface_backwardrQ  Úreflection_pad2d_backwardr-  Úreplication_pad2d_backwardrX  rf  Úreflection_pad3drh  Úreplication_pad3drk  Úreflection_pad3d_backwardÚreplication_pad3d_backwardrs  Ú_pdist_forwardrZ   rw  Ú_pdist_backwardr}  Úbaddbmmr–  Ú	bernoullir™  Ú
bernoulli_rœ  rt  rŸ  Úpoissonr¡  Ú_fused_moving_avg_obs_fq_helperr²  Úmmr»  r½  rð   rÜ  rã  Úmiopen_batch_normrñ  Úconvolutionrø  rm
  Ú_has_mkldnnrq
  rù  Ú_convolution_pointwiser   Ú_linear_pointwiser  Úhas_mklrs
  rò   Ú_mkl_linearr  rt
  r	  Úqconv2d_pointwiseÚqconv_pointwiserb  r  ÚbinaryÚbinary_tensorr!  Úqlinear_pointwiser&  r+  Úlinear_dynamic_fp16Úlinear_relu_dynamic_fp16r-  ru
  r.  Ú
max_pool2dr9  Úint4mm_packed_weight_cpurI  rN  Úquantize_per_tensorrP  Ú
avg_pool2drt  ry  Úavg_pool2d_backwardr�  Ú
avg_pool3dr›  Úavg_pool3d_backwardr¦  Ú_adaptive_avg_pool2dr«  Ú_adaptive_avg_pool3dr®  Ú_adaptive_avg_pool2d_backwardr¸  Ú_adaptive_avg_pool3d_backwardr½  r»  Úadaptive_max_pool2drÍ  rÑ  rÓ  Úadaptive_max_pool3drÚ  rÜ  rÝ  Úrepeat_interleaverà  rq   rå  rê  rï  r“   Ú_unsafe_indexr  Úconvolution_backwardr  Úaddbmmr%  Úrandint_liker)  Ú_fused_adam_Ú_fused_adamw_r@  Ú_fused_adamrF  Ú_int_mmrM  Ú_convert_weight_to_int4packrX  Ú#_convert_weight_to_int4pack_for_cpur]  Ú_weight_int4pack_mmrg  Ú_weight_int4pack_mm_for_cpurm  rv  rx  r¤  Ú_dyn_quant_pack_4bit_weightr®  Ú_dyn_quant_matmul_4bitr´  Ú_weight_int8pack_mmr»  Ú_cdist_forwardrÒ  Ú_cdist_backwardrÜ  Ú_embedding_bagr   Ú_embedding_bag_forward_onlyr  r  Únansumr  ÚmedianÚ	nanmedianr	  Ú
dim_valuesrŒ  r³   r  Úlogical_not_r  Úrepeatr  Úzero_r  Úmul_ÚScalarÚdiv_Úlogical_and_Úlogical_or_Úlogical_xor_r  Úadd_Úsub_r%  r?  Úsubr*  ÚroundÚdecimalsr,  r2  Ú
__rshift__r6  Ú
__lshift__r:  Úzeror<  r  r?  ÚfillrA  Úrelu_rD  Ú	_add_relurF  Úrrelu_with_noiserK  Úrrelu_with_noise_functionalrM  Úrrelu_with_noise_rO  Ú	index_putÚ_unsafe_index_putrS  Úmasked_fill_rW  Ú_masked_scalerZ  Úmasked_scatter_r^  Úmasked_scatterr`  Úmasked_scatter_backwardrb  Ú
index_put_rd  rs  Úbmmrv  Ú	dtype_outrx  r{  r€  rh  ri  r�  r§  r¢  r3  Ú max_pool2d_with_indices_backwardr´  Úmax_pool2d_with_indicesr¶  Úfractional_max_pool2drÊ  Úmax_pool3d_with_indicesrÖ  Ú max_pool3d_with_indices_backwardrÝ  rç  ré  ró  Úgrid_sampler_2d_backwardrû  r  r  r  r×  Úonesr  Úzerosr  Úselect_scatterr  Úslice_scatterr   r  r¸   r'  r/  Úgatherr5  rD  rK  rM  rW  rZ  Úscatter_addr_  Úscatter_add_rb  rY  r\  rV  r�  Úvalue_reducerf  Úscatter_rh  Ú#_scaled_dot_product_flash_attentionr}  r‚  rŠ  Ú#_scaled_dot_product_cudnn_attentionr’  Ú0_scaled_dot_product_fused_attention_overrideabler•  Ú9_scaled_dot_product_fused_attention_overrideable_backwardr¡  Ú,_scaled_dot_product_flash_attention_backwardr£  Ú+_scaled_dot_product_flash_attention_for_cpur¦  Ú4_scaled_dot_product_flash_attention_for_cpu_backwardrª  Ú*_scaled_dot_product_attention_math_for_mpsr¾  Ú'_scaled_dot_product_efficient_attentionrÂ  Ú0_scaled_dot_product_efficient_attention_backwardrÉ  Ú,_scaled_dot_product_cudnn_attention_backwardrË  Ú_flash_attention_forwardrÔ  Ú+_flash_attention_forward_no_dropout_inplacerÖ  rØ  Ú_flash_attention_backwardrÝ  Ú_efficient_attention_forwardré  Ú_efficient_attention_backwardÚSymIntrñ  r:  Ú
_scaled_mmr<  rˆ  Ú_scaled_mm_v2r‹  Úscatter_reduceÚtwoÚtwo_outr�  Úscatter_reduce_r‘  Úmultinomialr–  rš  r¨  r°  Ú_upsample_nearest_exact1dr¹  Ú_upsample_nearest_exact2drÁ  Ú"_upsample_nearest_exact2d_backwardrÅ  Ú_upsample_nearest_exact3drÇ   rÇ  Úvalues_stablerÊ  rà  Ú_thnn_fused_lstm_cellræ  rü  rÿ  r		  r	  r	  r	  Úargminr	  r	  Útopkr	  Ú_segment_reduce_backwardr"	  Úkthvaluer&	  r°   r8	  r5	  r<	  r?	  Úpixel_shufflerG	  rY	  Ú	bucketizeÚ
Tensor_outr`	  Ú
Scalar_outrb	  Úhistcrm	  Ú_upsample_bilinear2d_aaÚ_upsample_bicubic2d_aaÚ_upsample_lanczos2d_aarq	  Ú _upsample_bilinear2d_aa_backwardÚ_upsample_lanczos2d_aa_backwardru	  r|	  r�	  rÍ  rˆ	  Úsearchsortedr�	  r•	  Úembedding_dense_backwardr˜	  Ú_embedding_bag_backwardr�	  Ú_embedding_bag_dense_backwardr›	  Ú*_embedding_bag_per_sample_weights_backwardr¥	  Úisinr¬	  Ú	polygammar¯	  Ú_local_scalar_denser±	  r³	  rµ	  rÁ	  r
  Ú_grouped_mmr
  Ú_scaled_grouped_mmr
  Ú_foreach_normr
  Ú_softmaxr
  Úconstant_pad_ndr)
  r-
  Ú_jagged_to_padded_dense_forwardr1
  r9
  Ú_native_multi_head_attentionrH
  rK
  Úspecial_airy_aiÚspecial_bessel_y0Úspecial_bessel_y1Úspecial_modified_bessel_i0Úspecial_modified_bessel_i1Úspecial_modified_bessel_k0Úspecial_modified_bessel_k1Ú!special_scaled_modified_bessel_k0Ú!special_scaled_modified_bessel_k1Úspecial_chebyshev_polynomial_tÚspecial_chebyshev_polynomial_uÚspecial_chebyshev_polynomial_vÚspecial_chebyshev_polynomial_wÚ&special_shifted_chebyshev_polynomial_tÚ&special_shifted_chebyshev_polynomial_uÚ&special_shifted_chebyshev_polynomial_vÚ&special_shifted_chebyshev_polynomial_wÚspecial_hermite_polynomial_hÚspecial_hermite_polynomial_heÚspecial_laguerre_polynomial_lÚspecial_legendre_polynomial_prQ
  rU
  rZ
  r]
  Úlerp_Úaddcmul_Úaddcdiv_Útorch._refs.nn.functionalÚtorch._refs.specialr|
  r4   r:   r8   ú<module>rŒ     s:Q  ðã ß .Ý Ý Ý Ý 'ã Ý #ß +Ñ +÷ó õ "ß U÷÷ ÷ õ ÷õ ÷ <Ý 7ß 8Ý )ñ ˆTƒ]€Ùˆtƒ_€à‡y�y‡~�~€à*/¯-©-×*?Ñ*?ÀÈÐPVÓ*WÐ 'Ù % a£Ñ €ˆ)�Xòò"ð

˜ 8¨B°¨FÑ#3Ð"4°h¸rÀ2¸vÑ6FÐ"FÑGó 
ðà3óò(*òñ �—‘˜tŸ}™}Ð-Ó.Ùƒð
 
Ø
ØØ�=‰=ØØò7ó ó /ð7ñt �—	‘	×!Ñ! 4§9¡9§=¡=Ð1Ó2Ùƒñ'ó ó 3ð'ñ �×!Ñ!×)Ñ)¨4×+<Ñ+<×+@Ñ+@ÐAÓBÙƒØ%'ó %ó ó Cð%ñ$ ˆt×%Ñ%Ó&ÙƒñIó ó 'ðIñ Ø	‡[�[×Ñ˜$Ÿ+™+Ÿ/™/¨4¯;©;×+>Ñ+>ÀÇÁÇÁÐPóñ ˆX�yÓ!ñó "óðñ �×!Ñ!×)Ñ)¨4×+<Ñ+<×+@Ñ+@ÐAÓBÙƒñIó ó CðIò.òb�Vð  $ s¡)ð ¸4ó ñ �—‘×%Ñ% t§}¡}×'8Ñ'8Ð9Ó:Ùƒñ"<ó ó ;ð"<ðJ €ð $ s¡)ó ñ �—‘×%Ñ% t§}¡}×'8Ñ'8Ð9Ó:Ùƒñ@
ó ó ;ð@
ñF ˆt�}‰}×*Ñ*Ó+Ø"&ó 3ó ,ð3ñ ˆt�}‰}×$Ñ$Ó%ð �*‰*ØØØó
ó &ð
ñ �—‘×$Ñ$ d§l¡l×&6Ñ&6Ð7Ó8Ùƒð
 �*‰*ØØØóó ó 9ðñ& �—‘× Ñ  $§,¡,×"6Ñ"6Ð7Ó8Ùƒð �*‰*ØØØóó ó 9ðñ& �—	‘	×!Ñ! 4§9¡9§=¡=Ð1Ó2ÙƒØ%)°$¸tÐPTó ó ó 3ðñ ˆt×%Ñ%×-Ñ-Ó.ñ8ó /ð8ñ" ˆt×'Ñ'×/Ñ/Ó0ñ	!ó 1ð	!ò%
ñP ˆt×#Ñ#×+Ñ+Ó,óó -ðñ
 ˆt×$Ñ$×,Ñ,Ó-óó .ðñ
 �—‘×%Ñ% t§}¡}×'8Ñ'8Ð9Ó:Ùƒð*L�vð *L D¨¡Ið *L¸cð *LÈCò *Ló ó ;ð*LñZ ˆt�z‰z×!Ñ!Ó"óó #ðò:	(ñ ˆt�Š×&Ñ&Ó'ñó (ðñ ˆt×2Ò2Ó3ð
 Ø"&Ø$(ò)Øð)àð)ð ð)ð �4‰-ð	)ð
 ˜4‘Zð)ð �{Š{˜TÑ!ò)ó 4ð)ñX ˆt×.Ò.Ó/ð
 %)ò	Ø
ðàðð ðð �{Š{˜TÑ!ò	ó 0ðñ> ˆt×1Ò1Ó2ð Ø	
Ø$(ò)Øð)à
ð)ð ð)ð ð	)ð �{Š{˜TÑ!ò)ó 3ð)ñX ˆt×#Ò#Ó$ð ØØ$(Ø"ØØØò;<Ø—,‘,ð;<à�\‰\ð;<ð �4‰-ð;<ð �D‰=ð	;<ð
 �{Š{˜TÑ!ð;<ð ð;<ð ð;<ð ð;<ð ò;<ó %ð;<ñ| ˆt× Ò ×(Ñ(Ó)ð ò	IØ
ð	Ià	ð	Ið ð	Ið �L‰Lð		Ið
 ð	Ið ð	Ið ò	Ió *ð	Iñ ˆt×!Ò!×)Ñ)Ó*ð ò	Ø
ð	à	ð	ð ð	ð �L‰Lð		ð
 ð	ð ð	ð ò	ó +ð	ñ ƒÙˆt× Ò ×(Ñ(Ó)ñ'ó *ó ð'ñ ˆt×"Ò"×*Ñ*Ó+ð
 "Ø!Ø!ØØØò WØ
ð Wàð Wð �d‰]ð	 Wð
 �d‰]ð Wð �d‰]ð Wð ð Wð ð Wð ò Wó ,ð WñF �—’× Ñ  $§(¢(×"4Ò"4Ð5Ó6Ùƒñó ó 7ðñ ˆt�xŠx�|Š|Óóó ðñ �—’× Ñ  $§(¢(×"4Ò"4Ð5Ó6Ùƒñó ó 7ðñ ˆt�xŠx�|Š|Óóó ðñ ˆt�zŠz×!Ñ!Ó"ñ6ó #ð6ñ ˆt�zŠz�~‰~Óñ(ó ð(ñ
 ˆt×!Ò!×)Ñ)Ó*ñó +ðñ ˆt×!Ò!×%Ò%Ó&ñó 'ðñ ˆt�{Š{×"Ñ"Ó#ñó $ðñ ˆt×#Ò#×+Ñ+Ó,ð ØØØØó*ó -ð*ñ ˆt×'Ò'×/Ñ/Ó0ó,ó 1ð,ñ ˆt×3Ò3×;Ñ;Ó<óó =ðñ
 ˆt×0Ò0×8Ñ8Ó9ó6ó :ð6ñ& ˆt×<Ò<×DÑDÓEñó Fðñ
 ˆt×,Ò,×0Ò0Ó1ñó 2ðð

˜Fð 
¨Có 
ð   ð  ¨Fð  ¸#ó  ðN (,ò
Øð
àð
ð !%ó
ò$�Vð  Sð °Có ð
˜ð 
 Fð 
°$ð 
Àó 
ð"  ò	Øðàðð ðð ó	ð�Có ñ �×!Ò!×)Ñ)¨4×+<Ò+<×+HÒ+HÐIÓJÙˆ]˜NÓ+ò˜ð  sð ¸Tò ó ,ó Kðñ" �×$Ò$×,Ñ,¨d×.AÒ.A×.EÑ.EÐFÓGÙƒðB ð B¨6ò Bó ó HðBñ �—’Ð Ó!Ùˆ]˜NÓ+ð˜6ò ó ,ó "ðð Q ð Q¨Fó Qñ ˆt×*Ò*Ó+Ùƒð) ð )¨Fð )¸4ð )ÀFò )ó ó ,ð)ñ ˆt×"Ò"Ó#ÙƒòJ˜ð J Fð J°4ð JÀFò Jó ó $ðJñ ˆt�}Š}ÓÙƒò)�6ð ) $ð )°6ò )ó ó ð)ñ ˆt×$Ò$Ó%Ùƒò)˜6ð )¨$ð )¸6ò )ó ó &ð)ñ ˆt×&Ò&×.Ñ.Ó/ò˜&ð ¨ð ÀTò ó 0ðñ" Ø	×$Ò$×,Ñ,¨d×.MÒ.M×.QÑ.QÐRóñ ƒð. fð .°6ð .¸fò .ó óð.ñd ˆt×!Ò!×)Ñ)Ó*ò˜&ð °ò ó +ðñ �×)Ò)×1Ñ1°4×3LÒ3L×3PÑ3PÐQÓRÙˆT�8˜VÓ$ð Øò	Ø
ðð ðð ð	ð
 ˆ6�6˜6Ð!Ñ"òó %ó Sðñ& �×%Ò%×-Ñ-¨t×/DÒ/D×/HÑ/HÐIÓJÙƒð ò'Øð'àð'ð ð'ð
 ð'ð ò'ó ó Kð'ñT �—’×&Ñ&¨¯ª×(:Ñ(:Ð;Ó<ÙˆS�#�sÓØ/3ò �fð ¨ð ¸¸fÀfÈfÐ>TÑ8Uò ó ó =ðñ6 �×(Ò(×0Ñ0°$×2JÒ2J×2NÑ2NÐOÓPÙˆT�8˜VÓ$ð Øò	 Øð ð ð ð ð	 ð
 ˆ6�6˜6Ð!Ñ"ò ó %ó Qð ñF �×$Ò$×,Ñ,¨d×.BÒ.B×.FÑ.FÐGÓHÙƒð Øò4Øð4àð4ð ð4ð
 ð4ð ð4ð ò4ó ó Ið4ñn ˆt�~Š~ÓÙˆS�#�sÓð Øò	$Øð$àð$ð ð$ð ð	$ð
 ˆ6�6˜6Ð!Ñ"ò$ó ó ð$ðP˜ð   t¨T zÑ!2ó ñ* �—’×&Ñ&¨¯ª×(:Ñ(:Ð;Ó<ÙˆS�#Óò�fð  Cð ¸¸fÀf¸nÑ8Mò ó ó =ðñ4 �×$Ò$×,Ñ,¨d×.BÒ.B×.GÒ.GÐHÓIÙˆV�[ $¨Ó1ð'�vð ' %¨°¸ÀÐ(FÑ"Gò 'ó 2ó Jð'ñ$ ˆt×Ò×'Ñ'Ó(ð  ØØò	"Øð"àð"ð ð"ð �$‰Jò	"ó )ð"ðJ.Ø
ð.à
ð.ð ˆ4�‰9�d˜3‘iÐÑ ó.ð".Ø
ð.à
ð.ð �‰*ð.ð ˆ6�6ˆ>Ñó	.ð( fð °Vð Àó ñ ˆt×$Ò$Ó%ð
 ØØ ØØ Øò7Øð7àð7ð ð	7ð
 ð7ð �T‰Mð7ð 	�‰ð7ð �T‰Mð7ð �4‰-ð7ð ˆ6�6˜6 6Ð)Ñ*ò7ó &ð7ñt �×,Ò,×4Ñ4°d×6RÒ6R×6VÑ6VÐWÓXð ØØòØðàðð ð	ð
 ðð ðð 
�$‰ðð òó Yðñ4 ˆt×$Ò$Ó%ÙˆS�# 4Ô(ð ØØò+(Ø
ð+(àð+(ð ð+(ð ð	+(ð
 ð+(ð ˆ6�6ˆ>Ñò+(ó )ó &ð+(ñ^ ˆt×Ò×'Ñ'Ó(ñ
ó )ð
ñ ˆt�zŠzÓÙƒð
 ØòWØðWà	ðWð ðWð ð	Wð
 ðWð òWó ó ðWòtò>#;ñL ˆt×$Ò$Ó%Ùƒñ=ó ó &ð=ñ ˆt×%Ò%Ó&Ùƒñ>ó ó 'ð>ò(ñ< ˆt×-Ò-Ó.Ùˆ\ÓñSó ó /ðSñ ˆt×.Ò.Ó/Ùˆ\ÓñTó ó 0ðTò2Eñj ˆt×$Ò$Ó%Ùƒñ=ó ó &ð=ñ ˆt×%Ò%Ó&Ùƒñ>ó ó 'ð>ñ Ø×(Ò(×0Ñ0óñóðñ à×&Ò&×.Ñ.Ø×&Ò&×1Ò1Ø×'Ò'×/Ñ/Ø×'Ò'×2Ò2ð	óñ ˆ\Óñ&ó óð&ò:<Gñ~ ˆt×$Ò$Ó%Ùƒñ=ó ó &ð=ñ ˆt×%Ò%Ó&Ùƒñ>ó ó 'ñ>ñ à×&Ò&×.Ñ.Ø×&Ò&×1Ò1Ø×'Ò'×/Ñ/Ø×'Ò'×2Ò2ð	óñ ˆ\Óñ((ó óñ((ñV ˆt×"Ò"Ó#Ùƒò

˜fð 

©ð 

°vò 

ó ó $ñ

ñ ˆt×#Ò#Ó$ÙƒðP˜vð P¨Vð P¹ð PÀfð PÐQWò Pó ó %ñPñ �—’×$Ñ$ d§l¢l×&6Ñ&6Ð7Ó8Ù˜ÔØ/0¸ó 'ó ó 9ñ'ñ: �—’×&Ñ&¨¯ª×(:Ñ(:Ð;Ó<ÙƒØ&*ó Ió ó =ñIñ
 ˆt�Š×$Ò$Ó%óó &ññ ˆt�~Š~×ÒÓ óIó !ñIñ
 �—’×$Ñ$ d§l¢l×&6Ñ&6Ð7Ó8Ùƒó"ó ó 9ñ"ñ ˆt×3Ò3×;Ñ;Ó<ð Øó*ó =ñ*ñ. ˆt�wŠwÓÙ˜Ôò3˜UŸ[š[¨4Ñ/ò 3ó ó ñ3ó,Bõ
ð* .2òDØ—,‘,ðDà�L‰LðDð �‰I˜‰OðDð �#‰Y˜‰_ð	Dð
 �3‰i˜#‰oðDð ðDð ðDð ˜‘I ‘O dÑ*ôDóNQñ ˆt×%Ò%×-Ñ-Ó.ð"$Ø—,‘,ð"$à�L‰Lð"$ð �,‰,˜Ñ
ð"$ð —,‘, Ñ%ð	"$ð
 —‘ Ñ$ñ"$ð ñ"$ñ !&ñ"$ñ ó"$ó /ñ"$ñJ ˆt×Ò×'Ñ'Ó(ð#Ø—,‘,ð#à�L‰Lð#ð �,‰,ð#ð �‰Ið	#ð
 �#‰Yð#ð �3‰ið#ð ð#ð ˜‘Ið#ð ó#ó )ñ#ðL 	‡8‚8×ÔØ9>¿¹×9NÑ9NÙ�&˜&ó:Ñ6ñ �5—9‘9×#Ò#×:Ò:×BÑBÓCòó Dññ, �5—9‘9×#Ò#×5Ò5×=Ñ=Ó>òSó ?ñSð
 ‡x‚x×ÓØ:?¿-¹-×:OÑ:OÙ�6˜6ó;
Ñ7ñ 
�u—y‘y—}’}×0Ò0Ó	1ò	ó 
2ñ	ð
 :?¿¹×9NÑ9NÙ�&˜&ó:Ñ6ñ �5—9‘9×#Ò#×5Ò5×=Ñ=Ó>Ù�5—9‘9×#Ò#×3Ò3×;Ñ;Ó<Ù�5—9‘9×#Ò#×3Ò3×:Ò:Ó;ò4ó <ó =ó ?ñ4ñl �5—9‘9×#Ò#×5Ò5×<Ò<Ó=Ù�5—9‘9×#Ò#×5Ò5×CÒCÓDòó Eó >ññ< �5—9‘9×#Ò#×5Ò5×=Ñ=Ó>Ù�5—9‘9×#Ò#×5Ò5×<Ò<Ó=òó >ó ?ññ> �5—9‘9×#Ò#×5Ò5×<Ò<Ó=Ù�5—9‘9×#Ò#×5Ò5×CÒCÓDò$ó Eó >ñ$ñL �5—9‘9×#Ò#×7Ò7×?Ñ?Ó@Ù�5—9‘9×#Ò#×<Ò<×DÑDÓEò	ó Fó Añ	ð =B¿M¹M×<QÑ<QÙ�V˜Vó=Ñ9ñ �5—9‘9×&Ò&×1Ò1Ó2ð ØØØô
ó 3ñ
ñ8 �5—9‘9×&Ò&×?Ò?Ó@ò@ó Añ@ô,ñ ˆt×'Ò'Ó(ð#Ø�<‰<ñ#Ù %ñ#Ø36ñ#Ø?D¿{º{ð#à
‡\�\ó#ó )ñ#ñ ˆt�Š×&Ñ&Ó'ð ØØØØôMó (ñMôb(<ñX ˆt×'Ò'×/Ñ/Ó0òEó 1ñEñP ˆt�ŠÓÙƒð ØØØØôUJó ó  ñUJñp ˆt×'Ò'Ó(Ùˆ\ÓòK(ó ó )ñK(ñ\ ˆt×(Ò(×0Ñ0Ó1òó 2ññ" ˆt×(Ò(×0Ñ0Ó1ò@ó 2ñ@ñ ˆt×1Ò1×9Ñ9Ó:òFó ;ñFñ, ˆt×1Ò1Ó2Ùˆ\ÓòPó ó 3ñPñ
	
°6ð 	
ÀSõ 	
ñ ˆt×'Ò'Ó(ÙˆU�IÓò+ó ó )ñ+ñ\ ˆt×0Ò0Ó1Ùˆ\ÓòHó ó 2ñHñ$ ˆt×'Ò'Ó(ÙˆU�IÓò'ó ó )ñ'ñT ˆt×0Ò0Ó1Ùˆ\Óò(ó ó 2ñ(ñ
 ˆt×%Ò%×,Ñ,Ó-ô*ó .ñ*ñ �—’×$Ñ$ d§l¢l×&6Ñ&6Ð7Ó8Ùƒò
ó ó 9ñ
ñ �×#Ò#×+Ñ+¨T×-@Ò-@×-DÑ-DÐEÓFÙƒØ46ô 

¨có 

ó ó Gñ

ñ �—	‘	—‘×&Ò&×.Ñ.°·	±	·±×0FÒ0F×0JÑ0JÐKÓLÙƒòó ó Mññ" �—
’
×!Ñ! 4×#5Ò#5×#<Ñ#<Ð=Ó>òAó ?ñAñH �×)Ò)×1Ñ1Ð2Ó3ò.Hó 4ñ.Hñb �—’×#Ñ# T§[¢[§_¡_Ð5Ó6Ù˜ÔØ./°qô 'ó ó 7ñ'ñ0 �×!Ò!×(Ñ(Ð)Ó*ò'ó +ñ'ñ �×!Ò!×)Ñ)¨4×+=Ò+=×+EÑ+EÐFÓGð  Øõ!
ó Hñ
ñ2 �× Ò ×(Ñ(Ð)Ó*ð  Øõ!!ó +ñ!ñH �—’ˆ~ÓÙƒòBó ó ñBñ* �×0Ò0Ð1Ó2òó 3ññ& �×8Ò8Ð9Ó:òó ;ññ �×(Ò(Ð)Ó*ò8ó +ñ8ñ  �×0Ò0Ð1Ó2ò<ó 3ñ<ñ �×>Ò>Ð?Ó@ò<ó Añ<ñ"�3ñ "˜3ð " 3õ "ô@ñF �×0Ò0Ð1Ó2ðDØ!'¨$¡óDó 3ñDñ4 �×+Ò+Ð,Ó-ò;ó .ñ;ñ( �×(Ò(Ð)Ó*ò<ó +ñ<ñ ˆt×"Ò"×*Ñ*Ó+ò &ó ,ñ &ñF ˆt×#Ò#Ó$ÙƒòGó ó %ñGñ* ˆt×"Ò"×*Ñ*Ó+ð
 Ø	
ØØØØô`5ó ,ñ`5ñF ˆt×/Ò/×7Ñ7Ó8ò5ó 9ñ5ö
ñ �—’×#Ñ# T§[¢[§_¡_Ð5Ó6Ùƒñ=¸$õ =ó ó 7ñ=ñ �—’×#Ñ# T§^¢^×%;Ñ%;Ð<Ó=ò)ó >ñ)ñ à�Š�ŠØ�Š×ÒØ�Š×ÒØ�Š×!Ò!Ø�	Š	×ÑØ�	Š	×Òðó	ñ ˆX�yÓ!ôó "ó	ññ ˆt× Ò ×(Ñ(Ó)òó *ññ ˆt�{Š{×"Ñ"Ó#ò'ó $ñ'ñ& ˆt�zŠz×!Ñ!Ó"òó #ññ à�	Š	×ÒØ�	Š	×ÒØ�	Š	×ÑØ�	Š	×ÑØ×Ò×!Ñ!Ø×Ò× Ñ Ø×Ò×!Ñ!ðó
òó
ññ à�	Š	×ÒØ�	Š	×ÒØ�	Š	×ÑØ�	Š	×Ñð	óô*óñ*ñZ à�Š�ŠØ�Š�Šðóôóññ �—
’
×"Ñ" D§J¢J×$7Ò$7Ð8Ó9òó :ñô
ñ" �—’×&Ñ&¨¯ª×(>Ò(>Ð?Ó@òó Aññ �—’×&Ñ&¨¯ª×(>Ò(>Ð?Ó@òó Aññ ˆt�yŠy× Ñ Ó!ò&ó "ñ&ñ �—
’
×!Ñ! 4§:¢:×#4Ò#4Ð5Ó6òó 7ññ �—	’	× Ñ  $§)¢)×"2Ò"2Ð3Ó4ò"ó 5ñ"ñ ˆt�zŠz×!Ñ!Ó"òó #ññ ˆt�~Š~×$Ñ$Ó%Ùƒò¨Fó ó ó &ññ �×%Ò%Ð&Ó'ÙƒàRVô"ó ó (ñ"ñ �×0Ò0Ð1Ó2àRVô;ó 3ñ;ñ �×&Ò&Ð'Ó(àKOôó )ññ �—’×&Ñ&¨×(>Ò(>×(FÑ(FÐGÓHô"ó Iñ"ñ ˆt× Ò ×'Ò'Ó(òó )ññ
 ˆt×!Ò!×)Ñ)Ó*òó +ññ ˆt×#Ò#Ó$ò	ó %ñ	ñ ˆt×"Ò"Ó#Ùƒò6ó ó $ñ6ñ ˆt×+Ò+Ó,ò!ó -ñ!ñ ˆt�Š×&Ñ&Ó'ôó (ñö,ñ^ �—’× Ñ  $§(¢(§,¡,Ð/Ó0Ù˜Ôò5ó ó 1ñ5ñ �—’—’ §¢× 2Ò 2Ð3Ó4Ù˜ÔòJó ó 5ñJôôô6ô;ðh #ò-YØñYàñYð 	ñYð 	ñ	Yð
 	ñYð 	ñYð 	ñYð 	ñYð 	ñYð 	ñYð 	ñYð ñYð ñYð ñYð ñYð  ñ!Yð" ñ#Yð$ ñ%Yð& ñ'Yð( ð)Yð* ñ+Yð, õ-Yôx;4ð|38Øñ38àñ38ð ñ38ð 	ñ	38ð
 	ñ38ð 	ñ38ð 	ñ38ð 	ñ38ð 	ñ38ð 	ñ38ð 	ñ38ð 	ñ38ð ñ38ð ñ38ð ñ38ð  ñ!38ð" ñ#38ð$ ð%38ð& õ'38ôlI2ñX ˆt×4Ò4Ó5Ùˆ\Óò(ó ó 6ñ(ñV ˆt×+Ò+×3Ñ3Ó4ð ØØØô#ó 5ñ#ñL ˆt×)Ò)×1Ñ1Ó2òQó 3ñQñh ˆt×+Ò+Ó,ÙˆU�IÓð ØØØô_ó ó -ñ_ñD ˆt×4Ò4Ó5Ùˆ\Óò\ó ó 6ñ\ð~%
 Vñ %
°6õ %
öP˜tõ ð ñ ¨vñ È3õ ñ$ ˆt×,Ò,×4Ñ4Ó5ò#ó 6ñ#ñ$ ˆt×#Ò#Ó$Ùƒò8ó ó %ñ8ñ" ˆt×,Ò,Ó-Ùˆ\™;Ó'ò!ó (ó .ñ!ñ, �—	’	×!Ñ!Ð"Ó#ò.ó $ñ.ñ ˆt�Š×&Ñ&Ó'ð ØØØØô)ó (ñ)ñX �—	’	×!Ñ! 4§9¢9§=¡=Ð1Ó2Ùƒð ØØØØõó ó 3ññ( �—
’
×"Ñ" D§J¢J§N¡NÐ3Ó4Ùƒð ØØØØõó ó 5ññ( ˆt×"Ò"×*Ñ*Ó+ò&ó ,ñ&ñ ˆt×!Ò!×)Ñ)Ó*ô&ó +ñ&ô.ò˜ñ ¨Cñ ¸dõ ô /ô
ñ  ˆt�{Š{×"Ñ"Ó#ô'ó $ñ'öö6
ô ö
%öR4ñ ˆt×Ò×'Ñ'Ó(ò&ó )ñ&ñ
 ˆt× Ò Ó!òó "ññ
 à�Š×ÒØ�Š×ÒØ�Š×ÒØ�Š×!Ò!ð	óñ ƒô&ó óñ&ñ à�Š×ÒØ�Š×ÒØ�Š×ÒØ�Š×"Ò"ð	óôóññ �×8Ò8×@Ñ@ÐAÓBð
 ØØ#Øó<Øñ<à	ñ<ð ñ<ñ ñ	<ð
 ñ<ð ñ<ñ �4‰<ó<ó Cñ<ñ~ �×8Ò8×BÒBÐCÓDð ØØ#ØóØñà	ñð ñð ˜‰}ñ	ð
 ˜‰}ñð ˜‰}ññ ñð ñð ññ �4‰<óó Eññ4Øñà�S™#�X‰õñ& �×8Ò8Ð9Ó:ð ØØ#Øó(Øñ(à	ñ(ð ñ(ð ˜‰}ñ	(ð
 ñ(ñ ñ(ð ñ(ð ñ(ñ �4‰<ó(ó ;ñ(ñV �×EÒEÐFÓGð
  $ØØØ#Øó0Øñ0à	ñ0ð ñ0ð ˜‰}ñ	0ñ
 ñ0ð ñ0ð ñ0ñ �4‰<ó0ó Hñ0ñf ˆt×MÒMÓNð& ô%2Øñ2àñ2ð 
ñ2ð ñ	2ð
 ñ2ð ˜$‘Zð2ð 
ñ2ð ñ2ð ñ2ð ñ2ð ñ2ð ñ2ñ ñ2ð ñ2ð ñ2ð  ñ!2ñ$ �4‰<ó%2ó Oñ2ñ6 à×9Ò9ðóð( ó"Øñ"àñ"ð 
ñ"ð ð	"ð
 
ñ"ð ñ"ð ñ"ð ñ"ð ñ"ð ñ"ñ ñ"ð ñ"ð ñ"ð ñ"ñ �4‰<ó"óñ
"ñ. à×8Ò8ðóð ØØ#ØóØñà	ñð ññ ñ	ð
 ñð ˜‰}ññ �4‰<óóñ
ñ: à×AÒAðóð  $Øó!"Øñ!"àñ!"ð 
ñ!"ð ð	!"ð
 
ñ!"ð ñ!"ñ ñ!"ð ñ!"ð ˜‰}ñ!"ñ �4‰<ó!"óñ
!"ñH �×?Ò?Ð@ÓAð
  $ØØØ"&ØØó-Øñ-à	ñ-ð ñ-ð ˜‰}ñ	-ñ
 ñ-ð ñ-ð ˜4‘-ñ-ñ �4‰<ñ-ð ð-ð ˆ6�6ˆ>Ñó-ó Bñ-ñ` �×<Ò<Ð=Ó>ð ØØó))Øñ))à	ñ))ð ñ))ð ˜‰}ñ	))ð
 ñ))ð ñ))ñ �4‰<ó))ó ?ñ))ñX à×=Ò=ðóð" Øó4-Øñ4-àñ4-ð 
ñ4-ð ñ	4-ð
 ˜‰}ð4-ð 
ñ4-ð ñ4-ð ñ4-ð ñ4-ñ ñ4-ð ˜$‘Zñ4-ð ñ4-ñ �4‰<ó4-óñ
4-ñn à×9Ò9ðóð* ó!"Øñ"àñ"ð 
ñ"ð ð	"ð
 
ñ"ð ñ"ð ñ"ð ñ"ð ñ"ð ñ"ð ñ"ð ñ"ð ñ"ñ ñ"ð ñ"ñ  �4‰<ó!"óñ
"ñ0 à×%Ò%×-Ñ-ðóð  Ø#'Ø$(Ø#Ø"&Ø!%Ø!ó#JØñJà	ñJð ñJð ˜‰}ñ	Jð
 ˜‰}ñJð ñJð ñJñ ñJð ñJð ñJñ �4‰<ñJð ˜D‘jñJð ˜T‘zñJð ˜‰}ñJð ˜4‘-ñJð  ˜$‘ñ!Jð" �d‘
ó#Jóñ
JñZ �×@Ò@×HÑHÐIÓJð Ø#'Ø$(Ø#Ø"&Ø!%Ø!ò%&Ø	ñ&àñ&ð 
ñ&ð ñ	&ð
 ˜‰}ñ&ð ˜‰}ñ&ð ñ&ð ñ&ñ ñ&ð ñ&ð ñ&ñ �4‰<ñ&ð ˜D‘jñ&ð ˜T‘zñ&ð ˜‰}ñ&ð  ˜4‘-ñ!&ð" ˜$‘ñ#&ð$ �d‘
ó%&ó Kñ&ñR �×-Ò-×7Ò7Ð8Ó9ð Ø#'Ø$(Ø#Ø"&ó%'Øñ'à	ñ'ð ñ'ð ˜‰}ñ	'ð
 ˜‰}ñ'ð ñ'ð ñ'ñ ñ'ð ñ'ð ñ'ð ˜‰}ñ'ð ˜‰}ñ'ð ˜‰}ñ'ñ �4‰<ñ'ð ˜D‘jñ'ð  ˜T‘zñ!'ð" ˜‰}ñ#'ð$ ˜4‘-ó%'ó :ñ'ñT à×&Ò&ðóð( Ø#'Ø$(ó#,Øñ,àñ,ð 
ñ,ð ð	,ð
 
ñ,ð ñ,ð ñ,ð ñ,ð ñ,ð ñ,ñ ñ,ð ñ,ð ñ,ð ñ,ñ �4‰<ñ,ð  ˜D‘jñ!,ð" ˜T‘zó#,óñ
,ñ4 à×)Ò)ðóð   %ØØ%)Ø"Ø"ó/SØñ/Sà	ñ/Sð ð/Sð �4‰-ñ	/Sð
 ˜4‘-ñ/Sð ˜4‘-ñ/Sð ˜‘*ñ/Sð ˜‘*ñ/Sñ ñ/Sð ñ/Sð ñ/Sñ �4‰<ñ/Sð ˜d‘]ñ/Sð �t‰mñ/Sð �t‘ó/Sóñ
/Sñd à×*Ò*ðóð* Ø!%Ø"'ó%47Øñ47àñ47ð 
ñ47ð ð	47ð
 �4‰-ñ47ð ˜4‘-ñ47ð ˜4‘-ñ47ð —,’,ñ47ð —,’,ñ47ð ñ47ñ ñ47ð ñ47ð ñ47ð ñ47ð ñ47ñ  �4‰<ñ!47ð" ˜$‘Jñ#47ð$  ó%47óñ
47ðx !%Ø(,Ø$(Ø òAYØ
�,‰,ðAYà
�,‰,ñAYð �\‰\ñAYð �\‰\ð	AYð
 �,‰,˜Ñ
ñAYð —,‘, Ñ%ðAYð �{Š{˜TÑ!ñAYð õAYñH �—’×'Ñ'Ð(Ó)ð !%Ø(,Ø$(Ø òØ
�,‰,ðà
�,‰,ñð �\‰\ñð �\‰\ð	ð
 �,‰,˜Ñ
ñð —,‘, Ñ%ðð �{Š{˜TÑ!ñð óó *ñð, !%Ø$(Ø*.Ø*.Ø òiCØ
�,‰,ðiCà
�,‰,ñiCð �%—,‘,ÑñiCð ˜Ñ%ñ	iCð
 �%—,‘,ÑñiCð ˜Ñ%ðiCð �,‰,˜Ñ
ðiCð �{Š{˜TÑ!ñiCð �KÑ  4Ñ'ñiCð �KÑ  4Ñ'ñiCð õiCñX �×"Ò"×*Ñ*Ð+Ó,ð !%Ø'+Ø)-Ø òØ
�,‰,ðà
�,‰,ñð �%—,‘,Ññð ˜Ñ%ñ	ð
 �KÑ ñð �%—,‘,Ññð ˜Ñ%ñð �KÑ ðð �,‰,˜Ñ
ñð —+’+ Ñ$ñð ˜3‘i $Ñ&ñð óó -ññ: �×#Ò#×'Ò'¨×)<Ò)<×)DÒ)DÐEÓFÙƒô&ó ó Gñ&ñ
 ˆt×#Ò#×'Ò'Ó(ôó )ññ
 �× Ò ×(Ñ(¨$×*:Ò*:×*>Ñ*>Ð?Ó@Ùƒñ	Èô 	ó ó Añ	ôô,ñ* Ø	×Ò×$Ñ$ d×&DÒ&D×&LÑ&LÐMóô
óñ
ñ Ø	×Ò×$Ñ$ d×&DÒ&D×&LÑ&LÐMóôóññ. à×(Ò(×0Ñ0Ø×/Ò/×7Ñ7ðóð "Ø!óØñà˜# §¢Ñ,Ñ-ñð ˜˜uŸ|š|Ñ+Ñ,ññ �d‰lñ	ñ
 �d‰lóóññ: Ø	×Ò×$Ñ$ d×&DÒ&D×&LÑ&LÐMóô
óñ
ñ à�	Š	×ÑØ�	Š	×ÒØ�	Š	×ÒØ�	Š	×Òð	óôóñô*$ñN ˆt×)Ò)×1Ñ1Ó2ð
 Øôó 3ññ ˆt�Š×&Ñ&Ó'ò4/ó (ñ4/ñn ˆt�Š×&Ñ&Ó'ò)ó (ñ)ñX ˆt×$Ò$×,Ñ,Ó-ò$%ó .ñ$%ôN

ô
ñ �—’×#Ñ# T§[¢[×%8Ñ%8Ð9Ó:ô4ó ;ñ4ñ ˆt×!Ò!×)Ñ)Ó*ôó +ññ ˆt�yŠy× Ñ Ó!ô
Qó "ñ
Qñ ˆt×,Ò,Ó-ÙƒàLPôó ó .ññ" �—’×%Ñ% t§}¢}×';Ò';Ð<Ó=ÙˆX�yÓ!ôKó "ó >ñKð  #(×"9Ò"9Ñ ôQñ ˆt×7Ò7×?Ñ?Ó@ò	*ó Añ	*ñ ˆt×#Ò#×+Ñ+Ó,ò	0ó -ñ	0ñ ˆt×!Ò!×)Ñ)Ó*òó +ññ@ ˆt×-Ò-×5Ñ5Ó6òFó 7ñFñD �—’×%Ñ% t§~¢~×'@Ò'@ÐAÓBÙƒØ27¸uõ ó ó Cññ �—’×%Ò% t§~¢~×'@Ò'@ÐAÓBð
 Øó
Ø
ñ
àñ
ð ñ	
ð
 ó
ó Cñ
ñ �—
’
ˆ|ÓÙƒôEó ó ñEñ4 à×$Ò$×,Ñ,Ø×#Ò#×+Ñ+Ø×#Ò#×+Ñ+ðóð Øôóññ& à×-Ò-×5Ñ5Ø×,Ò,×4Ñ4ðóð Øôóññ8 ˆt×>Ò>×FÑFÓGòó Hññ$ �—’×'Ñ'¨¯ª×)<Ñ)<Ð=Ó>Ùƒô"ó ó ?ñ"ñ ˆu�y‰y�~‰~×(Ò(Ó)òó *ññ: ˆu�y‰y�~‰~× Ò Ó!ò6ó "ñ6ñ( ˆt× Ò Ó!Ùƒð
 Ø
Ø	Øõ-Kó ó "ñ-Kô`ñ ˆt×,Ò,Ó-òó .ññ ˆt×+Ò+Ó,ð ô'
ó -ñ'
ñT ˆt×1Ò1Ó2ð ôó 3ññ, ˆt×>Ò>Ó?ð ôó @ññ. ˆt�yŠyÓÙƒØ8=Àeõ 8ó ó ñ8ñ  ˆt�~Š~ÓÙƒñ6�cð 6 ð 6¨Fó 6ó ó ñ6ñ ˆt×'Ò'Ó(ðI &ó Ió )ñIñ ˆt�yŠyÓÙ˜Ôð"ˆvð "˜&ó "ó ó ñ"ñ ˆt�|Š|ÓÙƒð6�&ð 6˜Vó 6ó ó ñ6ô'ð^ ØØ(,Ø$(Ø óVKØñVKàñVKð �\‰\˜DÑ ñVKð �\‰\˜DÑ ñ	VKð
 �4‰-ðVKð �4‰-ñVKð —,‘, Ñ%ðVKð �{Š{˜TÑ!ñVKð õVKñr ˆt×ÒÓ Ùƒð ØØ$(óØñàñð �4‰-ðð �4‰-ð	ð
 �{Š{˜TÑ!ðð óó ó !ññ& �×'Ò'Ð(Ó)ð !%Ø $Ø(,Ø$(Ø óØ�<‰<ñà�<‰<ñð �\‰\ñð �\‰\ñ	ð
 �,‰,˜Ñ
ðð �,‰,˜Ñ
ñð —,‘, Ñ%ðð �{Š{˜TÑ!ñð óó *ññ6 ˆt×!Ò!×(Ò(Ó)ôó *ññ  ˆt�}Š}ÓÙƒñˆvð ˜Cñ °ð ¸ó ó ó ññ  ˆt×#Ò#Ó$Ùƒô3ó ó %ñ3ñl ˆt�~Š~ÓÙƒð Ø$Øò8Øð8àñ8ð ñ8ð ñ	8ð
 ð8ð ó8ó ó ñ8ñ. ˆt×3Ò3×;Ñ;Ó<ð
 ò	*Øð*à�&‰\ñ*ð �c‘ñ*ñ ó	*ó =ñ*ô,ñ ˆt×0Ò0×8Ñ8Ó9ð 
ØØØô-"ó :ñ-"ô`ò ˜d×2Ò2Ô 3Ú ˜d×4Ò4Ô 5Ú ˜d×4Ò4Ô 5Ú ˜d×=Ò=Ô >Ú ˜d×=Ò=Ô >Ú ˜d×=Ò=Ô >Ú ˜d×=Ò=Ô >Ú ˜d×DÒDÔ EÚ ˜d×DÒDÔ Eò ˜t×BÒBÔ CÚ ˜t×BÒBÔ CÚ ˜t×BÒBÔ CÚ ˜t×BÒBÔ CÚ ˜t×JÒJÔ KÚ ˜t×JÒJÔ KÚ ˜t×JÒJÔ KÚ ˜t×JÒJÔ KÚ ˜t×@Ò@Ô AÚ ˜t×AÒAÔ BÚ ˜t×AÒAÔ BÚ ˜t×AÒAÔ Bôñ ˆt�yŠyÓÙƒòó ó ññ$ ˆt�|Š|ÓÙƒØ./õ ó ó ññ ˆt�|Š|ÓÙƒØ./õ ó ó ñò, 	˜tŸyšyÓ)�Ú! $§,¢,Ó/�Ú! $§,¢,Ó/�ó
 Ü  Ü ôENòP …r:   