[j|}NUddlZddlmZddlmZddlmZmZmZm Z ddl Z ddl m Z ddl mZmZddlmZmZddlmZd d lmZmZmZd d lmZd d lmZmZgd ZGddeZGddeZ Gdde jBZ"Gdde jBZ#Gdde jBZ$de%de%de%de%de%de ede&ded e$fd!Z'd"eiZ(e)e*efe+d#<ie(d$d%d&Z,Gd'd(eZ-Gd)d*eZ.Gd+d,eZ/Gd-d.eZ0Gd/d0eZ1eed1e-jdf2dd3d4de e-de&ded e$fd5Z3eed1e.jdf2dd3d4de e.de&ded e$fd6Z4eed1e/jdf2dd3d4de e/de&ded e$fd7Z5eed1e0jdf2dd3d4de e0de&ded e$fd8Z6eed92dd3d4de e1de&ded e$fd:Z7 dAd;e%de%de*d?e&d d=f d@Z8y)BN) OrderedDict)partial)AnyCallable NamedTupleOptional)Conv2dNormActivationMLP)ImageClassificationInterpolationMode)_log_api_usage_once)register_modelWeights WeightsEnum)_IMAGENET_CATEGORIES)_ovewrite_named_paramhandle_legacy_interface) VisionTransformerViT_B_16_WeightsViT_B_32_WeightsViT_L_16_WeightsViT_L_32_WeightsViT_H_14_Weightsvit_b_16vit_b_32vit_l_16vit_l_32vit_h_14ceZdZUeed<eed<eed<ej Zedejfed<ejZ edejfed<y)ConvStemConfig out_channels kernel_sizestride. norm_layeractivation_layerN) __name__ __module__ __qualname__int__annotations__nn BatchNorm2dr&rModuleReLUr'd/var/www/html/ia-enhanced/venv/lib/python3.12/site-packages/torchvision/models/vision_transformer.pyr"r" sJ K+->>Jbii(913hsBII~.8r2r"c>eZdZdZdZdededeffd ZfdZxZ S)MLPBlockzTransformer MLP block.r in_dimmlp_dimdropoutcpt||||gtjd||j D]~}t |tj stjj|j|jTtjj|jdy)N)r'inplacer8ư>std) super__init__r-GELUmodules isinstanceLinearinitxavier_uniform_weightbiasnormal_)selfr6r7r8m __class__s r3r?zMLPBlock.__init__-s '6!2RWWVZdkl 6A!RYY'''166%GGOOAFFO5  6r2c |jdd}||dkrItdD];} dD]4} |d| dzd| } |d| zd| } | |vs!|j| || <6=t ||||||||y)Nversionr )rFrGlinear_r.)getrangepopr>_load_from_state_dict)rI state_dictprefixlocal_metadatastrict missing_keysunexpected_keys error_msgsrMitypeold_keynew_keyrKs r3rTzMLPBlock._load_from_state_dict6s!$$Y5 ?gk1X F.FD!'!uAdVr?riln_1r-MultiheadAttentionself_attentionDropoutr8ln_2r5mlp)rIrirjr7r8rkr&rKs r3r?zEncoderBlock.__init__Ysn "z*  33J Sdrvwzz'* z* J9r2inputc6tj|jdk(d|j|j |}|j |||d\}}|j |}||z}|j|}|j|}||zS)NrP2Expected (batch_size, seq_length, hidden_dim) got F) need_weights) torch_assertdimshapernrpr8rrrs)rIrtx_ys r3forwardzEncoderBlock.forwardns eiikQ&*\]b]h]h\i(jk IIe ""1a"?1 LLO I IIaL HHQK1u r2r(r)r*r`rr- LayerNormr+rbrrxr/r?Tensorrrcrds@r3rfrfVsw$6=R\\t5T::: :  : ! :S%((//12:* U\\ r2rfceZdZdZeej dfdededededed ed ed e d e j jfffd Z de jfdZxZS)Encoderz?Transformer Model Encoder for sequence to sequence translation.r;rg seq_length num_layersrirjr7r8rkr&.c t |tjt j d||j d|_tj||_ t} t|D]} t||||||| d| <tj| |_|||_y)Nrg{Gz?r<encoder_layer_)r>r?r- ParameterrxemptyrH pos_embeddingrqr8rrRrf Sequentiallayersln) rIrrrirjr7r8rkr&rr\rKs r3r?zEncoder.__init__}s  \\%++aZ*P*X*X]a*X*bczz'* .9mz" A+7! ,F^A3' ( mmF+ Z(r2rtctj|jdk(d|j||jz}|j |j |j|S)NrPrv)rxryrzr{rrrr8)rIrts r3rzEncoder.forwards\ eiikQ&*\]b]h]h\i(jk***wwt{{4<<#6788r2rrds@r3rrzsI6=R\\t5T))) )  )  ))!)S%((//12):9U\\9r2rc"eZdZdZddddeej ddfdeded ed ed ed ed ededede ede de j jfde e effd Zde j de j fdZde j fdZxZS)rz;Vision Transformer as per https://arxiv.org/abs/2010.11929.iNr;rg image_size patch_sizerrirjr7r8rk num_classesrepresentation_sizer&.conv_stem_configsc  t|t|tj||zdk(d||_||_||_||_||_ ||_ | |_ | |_ | |_ | tj} d}t!| D]g\}}| j#d|t%||j&|j(|j*|j|j,|j&}i| j#dtj.||d| |_ntj.d||| |_||zd z}tj2tj4dd||_|dz }t9|||||||| |_||_t?}| tj@|| |d <nItj@|| |d <tjB|d <tj@| | |d <tj||_"tG|j0tj.r|j0jH|j0j(dz|j0j(dz}tjJjM|j0jNtQjRd|z |j0jTJtjJjW|j0jTn|j0jXtG|j0jXtj.rtjJj[|j0jXjNdtQjRd|j0jXj&z |j0jXjT=tjJjW|j0jXjTt]|jDd rtG|jDj^tj@r|jDj^j`}tjJjM|jDj^jNtQjRd|z tjJjW|jDj^jTtG|jDjbtj@r{tjJjW|jDjbjNtjJjW|jDjbjTyy)Nrz&Input shape indivisible by patch size!rP conv_bn_relu_) in_channelsr#r$r%r&r' conv_lastr)rr#r$)rr#r$r%r head pre_logitsactr<rg@)meanr=)2r>r?rrxryrrrjr7rkr8rrr&r-r enumerate add_moduler r#r$r%r'Conv2d conv_projrzeros class_tokenrencoderrrrCTanhheadsrBrrD trunc_normal_rFmathsqrtrGzeros_rrHhasattrr in_featuresr)rIrrrrirjr7r8rkrrr&rseq_proj prev_channelsr\conv_stem_layer_configr heads_layersfan_inrKs r3r?zVisionTransformer.__init__sX D! j:-24\]$$$ !2 &#6 $  (}}HM-67H-I D))###A3'($1%;%H%H$:$F$F5<<#9#D#D)?)P)P  !7 C C  D   RYY=zghi )1DNYYJJWaDN!J.14 << Aq*(EFa            %4?M  &#%99Z#EL )+:?R)SL &"$'')L #%99-@+#NL ]]<0 dnnbii 0^^//$..2L2LQ2OORVR`R`RlRlmnRooF GG ! !$.."7"7TYYq6z=R ! S~~"".t~~223 ^^ % % 1jAYAY[][d[d6e GGOO((//ctyyt~~OgOgOtOtIt?u  ~~'',,8t~~77<<= 4::| ,DJJ">DIIaRXjDY ! Z GGNN4::0055 6 djjooryy 1 GGNN4::??11 2 GGNN4::??// 0 2r2r|returnc|j\}}}}|j}tj||jk(d|jd|dtj||jk(d|jd|d||z}||z}|j |}|j ||j||z}|jddd}|S)NzWrong image height! Expected z but got !zWrong image width! Expected rr r) r{rrxryrrreshaperjpermute) rIr|nchwpn_hn_ws r3_process_inputz VisionTransformer._process_input sWW 1a OO a4??*.KDOOK\\efgehhi,jk a4??*.J4??J[[defdggh,ij1f1f NN1  IIa#) 4 IIaA r2c|j|}|jd}|jj|dd}t j ||gd}|j |}|dddf}|j|}|S)Nrrrz)rr{rexpandrxcatrr)rIr|rbatch_class_tokens r3rzVisionTransformer.forward!s    " GGAJ!,,33Ar2> II(!,! 4 LLO adG JJqMr2)r(r)r*r`rr-rr+rbrrrxr/listr"r?rrrrcrds@r3rrsE#&-15`_'s training recipe.  num_paramsrr_metrics_ops _file_size_docsurl transformsrz>https://download.pytorch.org/models/vit_b_16_swag-9ac1b537.pthr resize_size interpolationi^-)rrg~jtSU@giX@gˡEK@g|?5^t@ These weights are learnt via transfer learning by end-to-end fine-tuning the original `SWAG `_ weights on ImageNet-1K data. rrrrrrzAhttps://download.pytorch.org/models/vit_b_16_lc_swag-4e70ced5.pth+https://github.com/pytorch/vision/pull/5793gbX9xT@gQ X@ These weights are composed of the original frozen `SWAG `_ trunk weights and a linear classifier learnt on top of them trained on ImageNet-1K data. rrrrrrrNr(r)r*rrr r IMAGENET1K_V1r BICUBIC_COMMON_SWAG_METAIMAGENET1K_SWAG_E2E_V1IMAGENET1K_SWAG_LINEAR_V1DEFAULTr1r2r3rr_s# G.#>  ""f##  ! M,% L +33    ""##  ! 4!( O +33    C""##  ! !6Gr2rc XeZdZedeediedddddd d id d d dZeZy)rz9https://download.pytorch.org/models/vit_b_32-d86f8d99.pthrri1BrzNhttps://github.com/pytorch/vision/tree/main/references/classification#vit_b_32rg|?5^R@gW@rgA`Т@gl u@rrrN r(r)r*rrr rrrr1r2r3rrs\ G.#>  ""f##  ! M,Gr2rc eZdZedeeddieddddd d d id d ddZedeeddejie dddddd iddddZ edeeddejie ddddddd id d d d!Z eZ y")#rz9https://download.pytorch.org/models/vit_l_16-852ce7e3.pthr)rri#rzNhttps://github.com/pytorch/vision/tree/main/references/classification#vit_l_16rg|?5^S@gFԨW@rgףp= N@g;O$@a These weights were trained from scratch by using a modified version of TorchVision's `new training recipe `_. rrz>https://download.pytorch.org/models/vit_l_16_swag-4f3808c9.pthri0)rrgjtV@gT㥛ĠX@gƟv@gy&11@rrzAhttps://download.pytorch.org/models/vit_l_16_lc_swag-4d563306.pthrgMbXIU@g^I [X@rrNrr1r2r3rrs% G.#3O  #"f##  " M.% L +33    #"##  " 4!( O +33    C#"##  " !6Gr2rc XeZdZedeediedddddd d id d d dZeZy)rz9https://download.pytorch.org/models/vit_l_32-c7638314.pthrri[ErzNhttps://github.com/pytorch/vision/tree/main/references/classification#vit_l_32rg|?5>S@gGzDW@rgK7.@gE@rrrNrr1r2r3rrs\ G.#>  #"f#"  " M,Gr2rc eZdZedeeddejiedddddd id d d d Z edeeddejieddddddd iddddZ e Z y)rz>https://download.pytorch.org/models/vit_h_14_swag-80465313.pthri%)rrrgS#V@g#~jX@rg~jŏ@gK7I@rrrzAhttps://download.pytorch.org/models/vit_h_14_lc_swag-c1eb923e.pthrri@%rgZd;OmU@gQnX@g= ףpd@gI k֢@rrN) r(r)r*rrr r rrrrrr1r2r3rr2s$ L +33    #"##  " 4!( O +33    C#"##  " !6%Gr2r pretrained)rT)rrc Rtj|}tdddddd||d|S)a Constructs a vit_b_16 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_B_16_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_B_16_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_B_16_Weights :members:   rrrirjr7rrr1)rverifyrrrrs r3rrkE(%%g.G      r2c Rtj|}tdddddd||d|S)a Constructs a vit_b_32 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_B_32_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_B_32_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_B_32_Weights :members: rrrrr1)rrrrs r3rrr r2c Rtj|}tdddddd||d|S)a Constructs a vit_l_16 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_L_16_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_L_16_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_L_16_Weights :members: rrr1)rrrrs r3rrE(%%g.G      r2c Rtj|}tdddddd||d|S)a Constructs a vit_l_32 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_L_32_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_L_32_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_L_32_Weights :members: r r rrrrr1)rrrrs r3rrrr2)rNc Rtj|}tdddddd||d|S)a Constructs a vit_h_14 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_H_14_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_H_14_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_H_14_Weights :members: r riirr1)rrrrs r3r r rr2r model_statezOrderedDict[str, torch.Tensor]interpolation_mode reset_headsc|d}|j\}}}|dk7rtd|j||zdzdz} | |k7r0|dz}| dz} |ddddddf} |ddddddf} | jddd} tt j |} | | z|k7rtd| | zd|| j d|| | } ||z} tjj| | |d }|j d|| }|jddd}tj| |gd }||d<|r;t}|jD]\}}|jd r|||<|}|S) aThis function helps interpolate positional embeddings during checkpoint loading, especially when you want to apply a pre-trained model on images with different resolution. Args: image_size (int): Image size of the new model. patch_size (int): Patch size of the new model. model_state (OrderedDict[str, torch.Tensor]): State dict of the pre-trained model. interpolation_mode (str): The algorithm used for upsampling. Default: bicubic. reset_heads (bool): If true, not copying the state of heads. Default: False. Returns: OrderedDict[str, torch.Tensor]: A state dict which can be loaded into the new model. zencoder.pos_embeddingrz%Unexpected position embedding shape: r NrzPseq_length is not a perfect square! Instead got seq_length_1d * seq_length_1d = z and seq_length = T)sizemode align_cornersrr)r{ ValueErrorrr+rrrr- functional interpolaterxrritems startswith)rrrrrrrrrjnew_seq_lengthpos_embedding_tokenpos_embedding_img seq_length_1dnew_seq_length_1dnew_pos_embedding_imgnew_pos_embeddingmodel_state_copykvs r3interpolate_embeddingsr*s* 78M - 3 3Az:Av@ATAT@UVWW J.14q8N #a !+Arr1H5)!QR(3.55aA>DIIj12 = (J 6bcptAdAcCCUV`Uab  .55a]Tab&*4!# 9 9 "# !:! !6 = =a^ \!6 = =aA F!II':r:sh #66 0H'66'B 9Z9+ s+ \!299!H#9bii#9LQ Qh    k " B&  d38n8KL{L^{4M{M`{46%{6%r,0@0N0N!OP6:T"23d]`evQ@,0@0N0N!OP6:T"23d]`evQ@,0@0N0N!OP6:T"23d]`evQ@,0@0N0N!OP6:T"23d]`evQ@!566:T"23d]`ev7H( KKK2K K  K & Kr2