a
    �ù»jA¸ ã                   @   s®
  d dl Z d dlZd dlmZ d dlmZmZmZ d dlZ	d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. d dl/m0Z0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z6 d dl7m8Z8 d dl9m:Z: d dl;m<Z<m=Z=m>Z>m?Z?m@Z@ d dlAmBZB d dlCmDZDmEZE d dlFmGZG �d(dd„ZHdd„ ZIdd„ ZJe
jK Lddd de	jMg¡dd„ ƒZNe
jK Ld d gd!fd dgdfg d"¢dfg¡d#d$„ ƒZOd%d&„ ZPd'd(„ ZQe
jK Rd)¡d*d+„ ƒZSe
jK Rd)¡d,d-„ ƒZTe
jK Rd)¡d.d/„ ƒZUd0d1„ ZVe
jK Ld2g d3¢e	 Wg d4¢g d5¢g d6¢g d7¢g¡fg d8¢g d9¢fg¡d:d;„ ƒZXe
jK Ld2g d<¢e	 Wg d7¢g d7¢g d=¢g d=¢g d>¢g¡fg d?¢g d@¢fg¡dAdB„ ƒZYdCdD„ ZZdEdF„ Z[dGdH„ Z\dIdJ„ Z]e
jK LdKeD¡e
jK LdLeE¡dMdN„ ƒƒZ^dOdP„ Z_e
jK LdQg dR¢¡dSdT„ ƒZ`dUdV„ ZadWdX„ Zbe
jK LdYe	 Wg dZ¢¡e	 Wg dZ¢¡d[œd\fe	 Wg d]¢¡e	 Wg d]¢¡d[œd^fe	 Wg d]¢¡e	 Wg dZ¢¡d[œd_fe	 Wg d]¢¡e	 Wg d`¢¡d[œdafe	 Wg dZ¢¡e	 Wg d]¢¡d[œdbfg¡dcdd„ ƒZce
jK Ldee	 Wg df¢¡e	 Wg dg¢¡d[œdhfg¡didj„ ƒZddkdl„ Zedmdn„ Zfe
jK Ldd de	jMg¡e
jK Ldod gd gfg¡e
jK Ldpe"ee#ddq�e,e-g¡drds„ ƒƒƒZge
jK Ldod gd gfg¡e
jK Ldpe"ee#ddq�e,e-g¡dtdu„ ƒƒZhdvdw„ Zidxdy„ Zjdzd{„ Zkd|d}„ Zle
jK Ld~dd€g¡d�d‚„ ƒZmdƒd„„ Zne
jK Ld…g d†¢¡d‡dˆ„ ƒZod‰dŠ„ Zpd‹dŒ„ Zqd�dŽ„ Zre
jKjLd�g d�fd‘d’gd“fgd”d•gd–�d—d˜„ ƒZse
jKjLd™dd dgg d"¢fg dš¢d–�d›dœ„ ƒZtd�dž„ Zue
jK LdŸg d ¢¡d¡d¢„ ƒZvd£d¤„ Zwd¥d¦„ Zxd§d¨„ Zyd©dª„ Zzd«d¬„ Z{d­d®„ Z|d¯d°„ Z}d±d²„ Z~d³d´„ Ze
jK Rd)¡dµd¶„ ƒZ€d·d¸„ Z�d¹dº„ Z‚d»d¼„ Zƒd½d¾„ Z„d¿dÀ„ Z…dÁdÂ„ Z†dÃdÄ„ Z‡e
jK LdÅdÆdÇg¡dÈdÉ„ ƒZˆe
jK Rd)¡dÊdË„ ƒZ‰e
jK Rd)¡dÌdÍ„ ƒZŠe
jK Rd)¡e
jK LdÎdÏdÆdÐe	jMe	jMfg¡dÑdÒ„ ƒƒZ‹e
jK LdÓdg¡e
jK Ld…g dÔ¢¡e
jK Ldd de	jMg¡dÕdÖ„ ƒƒƒZŒe
jK Ld…g dÔ¢¡d×dØ„ ƒZ�e
jK Ldd de	jMg¡dÙdÚ„ ƒZŽdÛdÜ„ Z�dÝdÞ„ Z�e
jK Ldd de	jMg¡dßdà„ ƒZ‘e
jK Lddd de	jMg¡dádâ„ ƒZ’e
jK Lddd de	jMg¡dãdä„ ƒZ“e
jK Lddd de	jMg¡dådæ„ ƒZ”dçdè„ Z•dédê„ Z–dëdì„ Z—dídî„ Z˜dïdð„ Z™dñdò„ Zšdódô„ Z›dõdö„ Zœd÷dø„ Z�dùdú„ Zždûdü„ ZŸe
jK LdŸe	j e	j¡e	j¢g¡dýdþ„ ƒZ£e
jK LdŸe	j e	j¡e	j¢g¡dÿ�d „ ƒZ¤e
jK Ldog �d¢g �d¢fg �d¢dd gd dgdd ggfg d"¢g �d¢g �d¢g �d¢gfg¡�d�d„ ƒZ¥�d�d„ Z¦�d�d	„ Z§�d
�d„ Z¨e
jK L�dg �d¢g �d¢fg �d¢g �d¢fg �d¢g �d¢fg¡�d�d„ ƒZ©e
jK Ldpe&e"ee#�ddq�e+e,e-eg¡e
jK L�dg �d¢¡�d�d„ ƒƒZªe
jK L�de	 Wd dg¡e	 Wdd g¡�dfe	 Wd dg¡e	 Wd dg¡�dfe	 Wd dg¡e	 Wd d g¡�dfe	 Wd d g¡e	 Wd d g¡�dfg¡�d�d„ ƒZ«e
jK L�de(e"e	jM�d�e(e#�de	jM�d�e(e,e	jM�d�e(e-e	jM�d�g¡�d �d!„ ƒZ¬�d"�d#„ Z­�d$�d%„ Z®�d&�d'„ Z¯dS ()  é    N)Úpartial)ÚchainÚpermutationsÚproduct)Úlinalg)Úhamming)Ú	bernoulli)ÚdatasetsÚsvm)Úmake_multilabel_classification)ÚUndefinedMetricWarning)Úaccuracy_scoreÚaverage_precision_scoreÚbalanced_accuracy_scoreÚbrier_score_lossÚclass_likelihood_ratiosÚclassification_reportÚcohen_kappa_scoreÚconfusion_matrixÚf1_scoreÚfbeta_scoreÚhamming_lossÚ
hinge_lossÚjaccard_scoreÚlog_lossÚmake_scorerÚmatthews_corrcoefÚmultilabel_confusion_matrixÚprecision_recall_fscore_supportÚprecision_scoreÚrecall_scoreÚzero_one_loss)Ú_check_targetsÚd2_log_loss_score)Úcross_val_score)ÚLabelBinarizerÚlabel_binarize)ÚDecisionTreeClassifier)ÚMockDataFrame)Úassert_allcloseÚassert_almost_equalÚassert_array_almost_equalÚassert_array_equalÚignore_warnings)Ú_nanaverage)ÚCSC_CONTAINERSÚCSR_CONTAINERS)Úcheck_random_stateFc                 C   s  | du rt  ¡ } | j}| j}|r:||dk  ||dk   }}|j\}}t |¡}tdƒ}| |¡ || ||  }}t	|d ƒ}tj
 d¡}tj|| |d| ¡f }tjdddd�}	|	 |d|… |d|… ¡ ||d… ¡}
|rò|
dd…d	f }
|	 ||d… ¡}||d… }|||
fS )
z¼Make some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    Né   é%   r   éÈ   ÚlinearT)ZkernelZprobabilityÚrandom_stateé   )r	   Ú	load_irisÚdataÚtargetÚshapeÚnpÚaranger1   ÚshuffleÚintÚrandomÚRandomStateZc_Zrandnr
   ZSVCÚfitZpredict_probaZpredict)ÚdatasetÚbinaryÚXÚyÚ	n_samplesÚ
n_featuresÚpÚrngZhalfZclfZy_pred_probaÚy_predÚy_true© rM   úOE:\soccer\ml_env\Lib\site-packages\sklearn/metrics/tests/test_classification.pyÚmake_prediction:   s*    


*rO   c               
   C   st  t  ¡ } t| dd�\}}}dddddœdd	d
ddœdddddœdddddœddddddœdœ}t||t t| jƒ¡| jdd�}| ¡ | ¡ ks’J ‚|D ]x}|dkrÊt	|| t
ƒs´J ‚|| || ksÈJ ‚q–||  ¡ ||  ¡ ksæJ ‚|| D ]}t|| | || | ƒ qîq–t	|d d t
ƒ�s(J ‚t	|d d t
ƒ�s@J ‚t	|d d tƒ�sXJ ‚t	|d d tƒ�spJ ‚d S ) NF©rC   rD   g§7½éMoê?gUUUUUUé?gh£¾³Qßé?é   )Ú	precisionÚrecallúf1-scoreÚsupportçUUUUUUÕ?gÆcŒ1Æ¸?g433333Ã?é   g³¦¬)kÊÚ?çÍÌÌÌÌÌì?ç“$I’$Iâ?é   gCÜFÁQà?g�¼cÕà?g¿��Æ¢ã?éK   ©rT   rR   rS   rU   gá?gDÖ~WGÞ?g]žè3«pà?)ÚsetosaZ
versicolorZ	virginicaú	macro avgÚaccuracyúweighted avgT)ÚlabelsÚtarget_namesÚoutput_dictr_   r]   rR   r^   rU   )r	   r8   rO   r   r<   r=   Úlenrb   ÚkeysÚ
isinstanceÚfloatr*   r?   )ÚirisrL   rK   Ú_Úexpected_reportÚreportÚkeyÚmetricrM   rM   rN   Ú,test_classification_report_dictionary_outputj   s`    üüüüüæ"û	rn   c                  C   sÞ   t g g dd�} dtjtjtjddœtjtjtjddœdœ}t| tƒsHJ ‚|  ¡ | ¡ ks\J ‚|D ]x}|dkr”t| | tƒs~J ‚| | || ksØJ ‚q`| |  ¡ ||  ¡ ks°J ‚|| D ]}t|| | | | | ƒ q¸q`d S )NT)rL   rK   rc   ç        r   r\   )r_   r^   r`   r_   )r   r<   Únanrf   Údictre   rg   r*   )rk   rj   rl   rm   rM   rM   rN   Ú2test_classification_report_output_dict_empty_input«   s,    üüørr   Úzero_divisionÚwarnr7   c                 C   s”   g d¢g d¢ }}t jdd��b}t||| dd� | dkrjt|ƒdksHJ ‚|D ]}d}|t|jƒv sLJ ‚qLn|rrJ ‚W d   ƒ n1 s†0    Y  d S )	N©ÚaÚbÚc)rv   rw   ÚdT©Úrecord)rs   rc   rt   r7   z7Use `zero_division` parameter to control this behavior.)ÚwarningsÚcatch_warningsr   rd   ÚstrÚmessage)rs   rL   rK   r{   ÚitemÚmsgrM   rM   rN   Ú0test_classification_report_zero_division_warningÉ   s    ÿr‚   zlabels, show_micro_avgT©r   r7   r2   c                 C   s\   ddgddg }}t ||| dd�}|r@d|v s2J ‚d|vsXJ ‚nd|v sLJ ‚d|vsXJ ‚dS )a3  Check the behaviour of passing `labels` as a superset or subset of the labels.
    WHen a superset, we expect to show the "accuracy" in the report while it should be
    the micro-averaging if this is a subset.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27927
    r   r7   T)ra   rc   z	micro avgr_   N©r   )ra   Zshow_micro_avgrL   rK   rk   rM   rM   rN   Ú1test_classification_report_labels_subset_supersetÙ   s    r…   c                  C   sÊ   t  g d¢g d¢g¡} t  g d¢g d¢g¡}t| |ƒdks>J ‚t| | ƒdksPJ ‚t||ƒdksbJ ‚t|t  |¡ƒdkszJ ‚t| t  | ¡ƒdks’J ‚t| t  | j¡ƒdks¬J ‚t|t  | j¡ƒdksÆJ ‚d S )N©r   r7   r7   ©r7   r   r7   ©r   r   r7   ç      à?r7   r   )r<   Úarrayr   Úlogical_notÚzerosr;   ©Úy1Úy2rM   rM   rN   Ú.test_multilabel_accuracy_score_subset_accuracyð   s    r�   c               	   C   s:  t dd�\} }}t| |d d�\}}}}t|ddgdƒ t|ddgdƒ t|d	d
gdƒ t|ddgƒ i ddifD ]Ä}t ¡ �¦ t d¡ t| |fi |¤Ž}t|ddƒ t| |fi |¤Ž}	t|	ddƒ t	| |fi |¤Ž}
t|
d
dƒ t
t| |fddi|¤Žd| |	 d| |	  dƒ W d   ƒ qp1 �s*0    Y  qpd S )NT©rD   ©Úaverageg\�Âõ(\ç?g333333ë?r2   g)\�Âõ(ì?gÃõ(\�Âå?çš™™™™™é?gR¸…ëQè?é   r“   rD   ÚerrorÚbetaé   é   )rO   r   r+   r,   r|   r}   Úsimplefilterr   r    r   r*   r   )rL   rK   ri   rI   ÚrÚfÚsÚkwargsÚpsÚrsÚfsrM   rM   rN   Ú%test_precision_recall_f1_score_binaryþ   s(    

ýr¢   z1ignore::sklearn.exceptions.UndefinedMetricWarningc                   C   s  dt ddgddgƒksJ ‚dtddgddgƒks4J ‚dtddgddgƒksNJ ‚dtddgddgdd�kslJ ‚dt ddgddgƒks†J ‚dtddgddgƒks J ‚dtddgddgƒksºJ ‚dtddgddgtdƒd�ksÜJ ‚tddgddgtdƒd�t tddgddgdd�¡k�sJ ‚d S )	Nç      ð?r7   r   ©r—   ro   éÿÿÿÿÚinfg     jø@)r   r    r   r   rg   ÚpytestÚapproxrM   rM   rM   rN   Ú+test_precision_recall_f_binary_single_class   s    "ÿr©   c               
   C   sâ  g d¢} g d¢}t | t d¡d�}t |t d¡d�}| |f||fg}t|ƒD ]˜\}\} }t| |g d¢d d�}tg d¢|ƒ t| |g d¢dd�}tt g d¢¡|ƒ d	D ]<}|d
kr¼|dkr¼q¦tt| |g d¢|d�t| |d |d�ƒ q¦qLdD ]Š}t 	t
¡�& t||t d¡|d� W d   ƒ n1 �s&0    Y  t 	t
¡�( t||t dd¡|d� W d   ƒ qê1 �sj0    Y  qêt g d¢g d¢g¡} t g d¢g d¢g¡}t| |d
ddgd�\}}	}
}tt ||	|
g¡t g d¢¡ƒ d S )N)r7   é   rª   r2   )r7   r7   rª   r2   r˜   ©Úclasses)r   r7   r2   rª   r™   ©ra   r“   )ro   r£   r£   r‰   ro   Úmacro)ÚmicroÚweightedÚsamplesr±   r   )Nr®   r¯   r±   é   r¥   r™   r†   ©r7   r   r   ©r7   r7   r7   r‡   r7   ©r“   ra   )ç      è?r7   ç«ªªªªªê?)r&   r<   r=   Ú	enumerater    r+   Úmeanr*   r§   ÚraisesÚ
ValueErrorrŠ   r   )rL   rK   Ú
y_true_binÚ
y_pred_binr9   ÚiÚactualr“   rI   r›   rœ   ri   rM   rM   rN   Ú$test_precision_recall_f_extra_labels3  s<    þ6ÿ(ÿrÀ   c            	      C   sè   g d¢} g d¢}t | t d¡d�}t |t d¡d�}| |f||fg}t|ƒD ]–\}\} }tt| |ddgd�}tt| |d d�}tdd	g|d d
�ƒ td|dd
�ƒ td|dd
�ƒ td|dd
�ƒ dD ]}||d
�||d
�ksÄJ ‚qÄqLd S )N)r7   r7   r2   rª   )r7   rª   rª   rª   r˜   r«   r7   rª   ©ra   r‰   r£   r’   r¶   r®   çUUUUUUå?r°   r¯   )r®   r°   r¯   )r&   r<   r=   r¸   r   r    r+   r*   )	rL   rK   r¼   r½   r9   r¾   Z	recall_13Z
recall_allr“   rM   rM   rN   Ú&test_precision_recall_f_ignored_labelsa  s    rÃ   c               	   C   s    t  g d¢g d¢g d¢g d¢g d¢g d¢g¡} t  g d¢g d¢g d¢g d	¢g d
¢g d¢g¡}d}tjt|d�� t| |dd� W d  ƒ n1 s’0    Y  dS )z:Test multiclass-multiouptut for `average_precision_score`.)r2   r2   r7   ©r7   r2   r   rƒ   ©r7   r2   r7   )r2   r   r7   ©çffffffæ?çš™™™™™É?çš™™™™™¹?©çš™™™™™Ù?ç333333Ó?rÌ   ©rÉ   r”   rÉ   ©rÈ   rÌ   r‰   )rË   rË   rÈ   )rÉ   rÈ   rÇ   z.multiclass-multioutput format is not supported©Úmatchr2   ©Ú	pos_labelN)r<   rŠ   r§   rº   r»   r   )rL   Úy_scoreÚerr_msgrM   rM   rN   Ú-test_average_precision_score_non_binary_classx  s*    úÿ
úÿ
rÕ   zy_true, y_score©r   r   r7   r2   rÆ   rÊ   rÍ   rÎ   )r   r   r   r   r7   r7   r7   r7   r7   r7   r7   )r   rÉ   rÉ   rË   r‰   ç333333ã?r×   rX   rX   r7   r7   c                 C   s   t | |ƒdksJ ‚dS )a(  
    Duplicate values with precision-recall require a different
    processing than when computing the AUC of a ROC, because the
    precision-recall curve is a decreasing curve
    The following situation corresponds to a perfect
    test statistic, the average_precision_score should be 1.
    r7   N©r   ©rL   rÓ   rM   rM   rN   Ú-test_average_precision_score_duplicate_values“  s    rÚ   )r2   r2   r7   r7   r   )rË   r‰   rÌ   )r”   r‰   rÌ   r†   )r‰   r‰   r×   c                 C   s   t | |ƒdksJ ‚d S )Nr£   rØ   rÙ   rM   rM   rN   Ú(test_average_precision_score_tied_values²  s    rÛ   c                  C   sN   d} t jt| d��( tg d¢g d¢ddd� W d   ƒ n1 s@0    Y  d S )NzšNote that pos_label \(set to 2\) is ignored when average != 'binary' \(got 'macro'\). You may use labels=\[pos_label\] to specify a single positive class.rÏ   rÅ   ©r7   r2   r2   r2   r®   ©rÒ   r“   )r§   ÚwarnsÚUserWarningr   ©r�   rM   rM   rN   Ú(test_precision_recall_f_unused_pos_labelÒ  s    ÿÿrá   c                  C   sD   t dd�\} }}dd„ }|| |ƒ |dd„ | D ƒdd„ |D ƒƒ d S )NTr‘   c                 S   s¤   t | |ƒ}t|ddgddggƒ | ¡ \}}}}|| ||  }t || ||  ||  ||  ¡}|dkrrdn|| }	t| |ƒ}
t|
|	dd� t|
ddd� d S )	Né   rª   é   é   r   r2   ©Údecimalç=
×£p=â?)r   r,   Úflattenr<   Úsqrtr   r+   )rL   rK   ÚcmÚtpÚfpÚfnÚtnÚnumZdenZtrue_mccZmccrM   rM   rN   Útestæ  s    
&
z*test_confusion_matrix_binary.<locals>.testc                 S   s   g | ]}t |ƒ‘qS rM   ©r~   ©Ú.0rF   rM   rM   rN   Ú
<listcomp>ô  ó    z0test_confusion_matrix_binary.<locals>.<listcomp>©rO   ©rL   rK   ri   rð   rM   rM   rN   Útest_confusion_matrix_binaryâ  s    
rø   c                  C   sD   t dd�\} }}dd„ }|| |ƒ |dd„ | D ƒdd„ |D ƒƒ d S )NTr‘   c                 S   s4   t | |ƒ}t|ddgddggddgddgggƒ d S )Nrä   rã   rª   râ   ©r   r,   )rL   rK   rê   rM   rM   rN   rð   û  s    
z5test_multilabel_confusion_matrix_binary.<locals>.testc                 S   s   g | ]}t |ƒ‘qS rM   rñ   rò   rM   rM   rN   rô      rõ   z;test_multilabel_confusion_matrix_binary.<locals>.<listcomp>rö   r÷   rM   rM   rN   Ú'test_multilabel_confusion_matrix_binary÷  s    
rú   c                  C   sJ   t dd�\} }}d	dd„}|| |ƒ |dd„ | D ƒdd„ |D ƒdd� d S )
NFr‘   c                 S   sü   t | |ƒ}t|ddgddggddgddggd	d
gddgggƒ |rJg d¢ng d¢}t | ||d�}t|ddgddggd	d
gddggddgddgggƒ |r g d¢ng d¢}t | ||d�}t|ddgddggd	d
gddggddgddggddgddgggƒ d S )Né/   r™   r˜   é   é&   r²   é   rª   é   r•   r2   é   )Ú0Ú2Ú1©r   r2   r7   rÁ   )r  r  r  Ú3)r   r2   r7   rª   r[   r   rù   )rL   rK   Ústring_typerê   ra   rM   rM   rN   rð     s&    
.ÿ.ÿüþz9test_multilabel_confusion_matrix_multiclass.<locals>.testc                 S   s   g | ]}t |ƒ‘qS rM   rñ   rò   rM   rM   rN   rô   #  rõ   z?test_multilabel_confusion_matrix_multiclass.<locals>.<listcomp>T)r  )Frö   r÷   rM   rM   rN   Ú+test_multilabel_confusion_matrix_multiclass  s    

r  Úcsc_containerÚcsr_containerc                 C   sÚ  t  g d¢g d¢g d¢g¡}t  g d¢g d¢g d¢g¡}||ƒ}||ƒ}| |ƒ}| |ƒ}t  g d¢¡}dd	gddggdd	gddggd	d
gdd	ggg}	|||g}
|||g}|
D ]"}|D ]}t||ƒ}t||	ƒ q´q¬t||dd�}t|dd	gddggddgd	dggd	dgd
d	gggƒ t||d
d	gd�}t|d	d
gdd	ggdd	gddgggƒ t||d
d	gdd�}t|d	d	gddggddgd	d	ggd	dgdd	gggƒ t|||dd�}t|d
d	gd
d
ggddgd	dggd	dgdd	gggƒ d S )Nr‡   ©r   r7   r   ©r7   r7   r   r³   r†   rˆ   )r2   r7   rª   r7   r   r2   T©Ú
samplewiserÁ   )ra   r  )Úsample_weightr  rª   r²   )r<   rŠ   r   r,   )r  r	  rL   rK   Z
y_true_csrZ
y_pred_csrZ
y_true_cscZ
y_pred_cscr  Zreal_cmZtruesÚpredsZ
y_true_tmpZ
y_pred_tmprê   rM   rM   rN   Ú+test_multilabel_confusion_matrix_multilabel&  s0    .


4&4ÿr  c                  C   sì  t  g d¢g d¢g d¢g¡} t  g d¢g d¢g d¢g¡}tjtdd��" t| |d	d
gd� W d   ƒ n1 sn0    Y  tjtdd��0 t| |g d¢g d¢g d¢gd� W d   ƒ n1 s¼0    Y  d}tjt|d��  t| |dgd� W d   ƒ n1 sþ0    Y  d}tjt|d��  t| |dgd� W d   ƒ n1 �sB0    Y  tjtdd��& tg d¢g d¢dd� W d   ƒ n1 �sˆ0    Y  d}tjt|d��2 tg d¢g d¢gg d¢g d¢gƒ W d   ƒ n1 �sÞ0    Y  d S )Nr‡   r
  r  r³   r†   rˆ   úinconsistent numbers of samplesrÏ   r7   r2   ©r  zshould be a 1d array©r7   r2   rª   )r2   rª   r™   )rª   r™   r˜   z%All labels must be in \[0, n labels\)r¥   rÁ   rª   zSamplewise metricsrƒ   rÄ   Tr  z'multiclass-multioutput is not supported)r2   r7   r   ©r7   r   r2   )r<   rŠ   r§   rº   r»   r   )rL   rK   rÔ   rM   rM   rN   Ú'test_multilabel_confusion_matrix_errorsP  s&    0ÿ$.06r  z%normalize, cm_dtype, expected_results))Útruerœ   çµùTUUÕ?)Úpredrœ   r  )Úallrœ   g��eÇq¼?)Nr¾   r2   c                 C   sL   g d¢d }t ttg d¢ƒŽ ƒ}t||| d�}t||ƒ |jj|ksHJ ‚d S )Nrƒ   r²   ©Ú	normalize)Úlistr   r   r   r)   ÚdtypeÚkind)r  Zcm_dtypeZexpected_resultsÚy_testrK   rê   rM   rM   rN   Útest_confusion_matrix_normalizen  s
    

r   c                  C   sÒ   g d¢} g d¢}t | |dd�}| ¡ t d¡ks4J ‚t ¡ �* t dt¡ t | |dd�}W d   ƒ n1 sl0    Y  | ¡ t d¡ksŒJ ‚t ¡ �* t dt¡ t || dd� W d   ƒ n1 sÄ0    Y  d S )	N)r   r   r   r   r7   r7   r7   r7   )r   r   r   r   r   r   r   r   r  r  ç       @r–   r  r£   )r   Úsumr§   r¨   r|   r}   rš   ÚRuntimeWarning)r  rK   Zcm_trueZcm_predrM   rM   rN   Ú,test_confusion_matrix_normalize_single_class  s    
,
r$  c                  C   sL   g d¢} g d¢}t jtdd�� t|| ƒ W d  ƒ n1 s>0    Y  dS )z8Test `confusion_matrix` warns when only one label found.©r   r   r   r   zA single label was found inrÏ   N)r§   rÞ   rß   r   )r  rK   rM   rM   rN   Ú"test_confusion_matrix_single_label’  s    r&  zparams, warn_msg)r   r   r   r   r   r   ©rL   rK   z2samples of only one class were seen during testing)r7   r7   r7   r   r   r   z:positive_likelihood_ratio ill-defined and being set to nanz+no samples predicted for the positive class©r   r   r   r7   r7   r7   z:negative_likelihood_ratio ill-defined and being set to nanz@no samples of the positive class were present in the testing setc                 C   s@   t jt|d�� tf i | ¤Ž W d   ƒ n1 s20    Y  d S ©NrÏ   )r§   rÞ   rß   r   )ÚparamsÚwarn_msgrM   rM   rN   Útest_likelihood_ratios_warnings›  s    1r,  zparams, err_msg)r   r7   r   r7   r   ©r7   r7   r   r   r2   zeclass_likelihood_ratios only supports binary classification problems, got targets of type: multiclassc                 C   s@   t jt|d�� tf i | ¤Ž W d   ƒ n1 s20    Y  d S r)  )r§   rº   r»   r   )r*  rÔ   rM   rM   rN   Útest_likelihood_ratios_errorsÐ  s    r.  c                  C   sÖ   t  dgd dgd  ¡} t  dgd dgd  dgd  ¡}t| |ƒ\}}t|dƒ t|d	ƒ t| | ƒ\}}t|t jd ƒ t|t  d¡d
d� t  dgd dgd  ¡}t| ||d�\}}t|dƒ t|dƒ d S )Nr7   rª   r   rä   r2   é
   rã   g«ªªªªªö?g_B{	í%ä?gê-�™—q=)Zrtolr£   é   ro   r˜   r  gUUUUUU@gÇqÇqÜ?)r<   rŠ   r   r)   r,   rp   rŒ   )rL   rK   ÚposÚnegr  rM   rM   rN   Útest_likelihood_ratiosæ  s    $


r3  c                  C   s¬  t  dgd dgd  ¡} t  dgd dgd  dgd  dgd  ¡}t| |ƒ}t|dd	d
� |t|| ƒksrJ ‚t  | dgd ¡} t  |dgd ¡}t| |ddgd�|ks°J ‚tt| | ƒdƒ t  dgd dgd  dgd  ¡} t  dgd dgd  dgd  ¡}tt| |ƒddd
� t  dgd dgd  dgd  ¡} t  dgd dgd  dgd  ¡}tt| |ƒddd
� tt| |dd�ddd
� tt| |dd�ddd
� d S )Nr   é(   r7   é<   rZ   r/  é2   gƒÀÊ¡EÖ?rª   rå   r2   r™   rÁ   r£   é.   é,   é4   é    é   gÉå?¤é?g+‡ÙÎí?r5   ©Úweightsg®Ø_vOî?Z	quadraticgœ¢#¹ü‡î?)r<   rŠ   r   r*   Úappend)rŽ   r�   ÚkapparM   rM   rN   Útest_cohen_kappaÿ  s&    .
$$$$ÿr@  zy_true, y_predrm   r¤   c                 C   sj   t  ¡ �( t  d¡ | |||d�}W d  ƒ n1 s60    Y  t |¡rZt |¡sfJ ‚n||ksfJ ‚dS )zmCheck the behaviour of `zero_division` when setting to 0, 1 or np.nan.
    No warnings should be raised.
    r–   ©rs   N)r|   r}   rš   r<   Úisnan)rm   rL   rK   rs   ÚresultrM   rM   rN   Ú!test_zero_division_nan_no_warning  s    

,
rD  c                 C   sH   t  t¡� | ||dd�}W d  ƒ n1 s.0    Y  |dksDJ ‚dS )ztCheck the behaviour of `zero_division` when setting to "warn".
    A `UndefinedMetricWarning` should be raised.
    rt   rA  Nro   )r§   rÞ   r   )rm   rL   rK   rC  rM   rM   rN   Útest_zero_division_nan_warning7  s    ,rE  c                  C   sN   t j d¡} | jdddd�}| jdddd�}tt||ƒt  ||¡d dƒ d S )Nr   r2   rZ   ©Úsize©r   r7   r/  )r<   r@   rA   Úrandintr*   r   Zcorrcoef)rJ   rL   rK   rM   rM   rN   Ú-test_matthews_corrcoef_against_numpy_corrcoefJ  s    ÿrJ  c            	         sÒ   t j d¡} | jdddd�}| jdddd�}|  d¡}t|||d�‰ tˆ ƒ‰t‡ ‡fdd„tˆƒD ƒƒ}t‡ ‡fdd„tˆƒD ƒƒ}t  ‡ ‡fd	d„tˆƒD ƒ¡}|t  	|| ¡ }t
|||d�}t||d
ƒ d S )Nr   r2   rZ   rF  r  c              	      sX   g | ]P}t ˆƒD ]B}t ˆƒD ]4}ˆ ||f ˆ ||f  ˆ ||f ˆ ||f   ‘qqqS rM   ©Úrange)ró   ÚkÚmÚl©ÚCÚNrM   rN   rô   `  s   ýz9test_matthews_corrcoef_against_jurman.<locals>.<listcomp>c                    s@   g | ]8‰ ˆd d …ˆ f   ¡ t  ‡‡‡ fdd„tˆƒD ƒ¡ ‘qS )Nc                    s.   g | ]&}t ˆƒD ]}|ˆkrˆ ||f ‘qqS rM   rK  ©ró   rœ   Úg©rQ  rR  rM  rM   rN   rô   j  rõ   úDtest_matthews_corrcoef_against_jurman.<locals>.<listcomp>.<listcomp>©r"  r<   rL  ©ró   rP  ©rM  rN   rô   h  s   þÿc                    s@   g | ]8‰ ˆˆ d d …f   ¡ t  ‡‡‡ fdd„tˆƒD ƒ¡ ‘qS )Nc                    s.   g | ]&}t ˆƒD ]}|ˆkrˆ ||f ‘qqS rM   rK  rS  rU  rM   rN   rô   q  rõ   rV  rW  rX  rP  rY  rN   rô   o  s   þÿr/  )r<   r@   rA   rI  Zrandr   rd   r"  rL  ré   r   r*   )	rJ   rL   rK   r  Zcov_ytypZcov_ytytZcov_ypypZ
mcc_jurmanZmcc_oursrM   rP  rN   Ú%test_matthews_corrcoef_against_jurmanT  s0    
þÿýÿýÿrZ  c                  C   s,  t j d¡} dd„ | jdddd�D ƒ}tt||ƒdƒ dd„ |D ƒ}tt||ƒd	ƒ t|d
dgd�}t  |d
d¡}tt||ƒd	ƒ ttg d¢g d¢ƒdƒ tt|d
gt|ƒ ƒdƒ g d¢}g d¢}tt||ƒdƒ dgd dgd  }t	 
t¡�$ tt|||d�dƒ W d   ƒ n1 �s0    Y  d S )Nr   c                 S   s   g | ]}|d krdnd‘qS )r   rv   rw   rM   ©ró   r¾   rM   rM   rN   rô   }  rõ   z*test_matthews_corrcoef.<locals>.<listcomp>r2   rZ   rF  r£   c                 S   s   g | ]}|d krdnd ‘qS )rv   rw   rM   r[  rM   rM   rN   rô   ƒ  rõ   r¥   rv   rw   r«   r%  ro   )r7   r   r7   r7   r   r7   r7   r7   r   r7   r7   r7   r7   r7   r7   r7   r   r7   r7   r7   )r7   r7   r7   r   r   r7   r7   r7   r7   r   r7   r7   r7   r   r7   r7   r7   r   r7   r7   r7   r/  r  )r<   r@   rA   rI  r*   r   r&   Úwhererd   r§   rº   ÚAssertionError)rJ   rL   Z
y_true_invZy_true_inv2Úy_1Úy_2ÚmaskrM   rM   rN   Útest_matthews_corrcoef{  s     ra  c            	         sL  t j d¡} tdƒ‰ d}‡ fdd„| jd|dd�D ƒ}tt||ƒdƒ g d	¢}g d
¢}tt||ƒdƒ g d	¢}g d¢}tt||ƒdt  d¡ ƒ g d¢}g d¢}tt||ƒdƒ g d¢}g d¢}tt||ƒdƒ g d¢}g d¢}tt||ƒdƒ g d¢}g d¢}g d¢}tt|||d�dƒ g d¢}g d¢}g d¢}tt|||d�dƒ d S )Nr   rv   r™   c                    s   g | ]}t ˆ | ƒ‘qS rM   )Úchrr[  ©Zord_arM   rN   rô   ¢  rõ   z5test_matthews_corrcoef_multiclass.<locals>.<listcomp>rZ   rF  r£   )r   r   r7   r7   r2   r2   )r2   r2   r   r   r7   r7   g      à¿)r7   r7   r   r   r   r   iôÿÿÿi€  rƒ   )rª   rª   rª   ro   ©	r   r7   r2   r   r7   r2   r   r7   r2   )	r7   r7   r7   r2   r2   r2   r   r   r   )r   r   r7   r7   r2   r-  ©r7   r7   r7   r7   r   r  r¥   rÖ   ©r7   r7   r   r   )r<   r@   rA   ÚordrI  r*   r   ré   )	rJ   Ú	n_classesrL   Z
y_pred_badZ
y_pred_minrK   r^  r_  r  rM   rc  rN   Ú!test_matthews_corrcoef_multiclassž  s@    ÿÿri  Ún_pointséd   i'  c                    s–   t j d¡‰ dd„ }‡ fdd„}t  ddg| ¡}tt||ƒdƒ t  g d¢| ¡}tt||ƒdƒ || ƒ\}}tt||ƒdƒ tt||ƒ|||ƒƒ d S )	NišÈ3c                 S   sx   t | |ƒ}|d }|d }|d }t| ƒ}|| | }|| | }|| ||  }	|| d|  d|  }
|	t |
¡ S )N©r7   r7   )r7   r   rH  r7   )r   rd   r<   ré   )rL   rK   Zconf_matrixZtrue_posZ	false_posZ	false_negrj  Zpos_rateZactivityZmcc_numeratorZmcc_denominatorrM   rM   rN   Úmcc_safeÛ  s    
z1test_matthews_corrcoef_overflow.<locals>.mcc_safec                    s8   ˆ   | ¡}|dˆ   | ¡d   }|dk}|dk}||fS )NrÈ   r‰   )Zrandom_sample)rj  Zx_trueZx_predrL   rK   ©rJ   rM   rN   Ú	random_ysç  s
    
z2test_matthews_corrcoef_overflow.<locals>.random_ysro   r£   )ro   r£   r!  )r<   r@   rA   Úrepeatr*   r   )rj  rm  ro  ZarrrL   rK   rM   rn  rN   Útest_matthews_corrcoef_overflowÖ  s    rq  c            
      C   s˜  t dd�\} }}t| |d d�\}}}}t|g d¢dƒ t|g d¢dƒ t|g d¢dƒ t|g d¢ƒ t| |d	d
d�}t|ddƒ t| |d
d�}t|ddƒ t| |d
d�}	t|	ddƒ t| |dd�}t|ddƒ t| |dd�}t|ddƒ t| |dd�}	t|	ddƒ t| |dd�}t|ddƒ t| |dd�}t|ddƒ t| |dd�}	t|	ddƒ t t	¡� t| |dd� W d   ƒ n1 �s€0    Y  t t	¡� t| |dd� W d   ƒ n1 �sº0    Y  t t	¡� t| |dd� W d   ƒ n1 �sô0    Y  t t	¡�  t
| |ddd� W d   ƒ n1 �s00    Y  t| |g d¢d d�\}}}}t|g d¢dƒ t|g d¢dƒ t|g d¢dƒ t|g d¢ƒ d S )NFr‘   r’   )ç�Âõ(\�ê?ç…ëQ¸Õ?gáz®GáÚ?r2   )çHáz®Gé?g
×£p=
·?rX   )çìQ¸…ëé?ç333333Ã?rç   )rQ   rW   rZ   r7   r¯   rÝ   gö(\�Âõà?r®   r×   gR¸…ëQà?r°   g®GázÞ?r±   r‰   ©r“   r—   r  r­   )rr  g=
×£p=Ú?rs  )rt  rX   rÉ   )ru  rç   rv  )rQ   rZ   rW   )rO   r   r+   r,   r   r    r   r§   rº   r»   r   )
rL   rK   ri   rI   r›   rœ   r�   rŸ   r    r¡   rM   rM   rN   Ú)test_precision_recall_f1_score_multiclassø  sN    ...0ÿrx  r“   )r±   r¯   r®   r°   Nc                 C   sv   t  g d¢g¡}t  g d¢g¡}t||g d¢g | d�\}}}}t|dƒ t|dƒ t|dƒ | d u rrt|g d¢ƒ d S )Nrf  ©r   r   r7   r7   )rª   r   r7   r2   )ra   Úwarn_forr“   r   ©r   r7   r7   r   )r<   rŠ   r   r,   )r“   rL   rK   rI   r›   rœ   r�   rM   rM   rN   Ú;test_precision_refcall_f1_score_multilabel_unordered_labels2  s    ÿ


r|  c            
      C   sä   t  g d¢¡} t  g d¢¡}t| |d d�\}}}}t| |dd�\}}}}|t  |¡ksZJ ‚|t  |¡kslJ ‚|t  |¡ks~J ‚t| |dd�\}}}}t  | ¡}	|t j||	d�ks´J ‚|t j||	d�ksÊJ ‚|t j||	d�ksàJ ‚d S )N)r   r7   r   r   r7   r7   r   r7   r   r   r7   r   r7   r   r7   )r7   r7   r   r7   r   r7   r7   r7   r7   r   r7   r   r7   r   r7   r’   r®   r°   r<  )r<   rŠ   r   r¹   Zbincountr“   )
rL   rK   rŸ   r    r¡   ri   rI   r›   rœ   rU   rM   rM   rN   Ú.test_precision_recall_f1_score_binary_averagedA  s    
r}  c               
   C   s–   t jdd�} zrt  g d¢¡}t  g d¢¡}tt||dd�ddƒ tt||dd�ddƒ tt||dd�ddƒ W t jf i | ¤Ž nt jf i | ¤Ž 0 d S )	NÚraise)r  )r   r7   r2   r   r7   r2   )r2   r   r7   r7   r2   r   r®   r’   ro   r2   )r<   ZseterrrŠ   r*   r   r    r   )Zold_error_settingsrL   rK   rM   rM   rN   Útest_zero_precision_recallR  s    r  c                  C   sš   t dd�\} }}t| |ddgd�}t|ddgddggƒ t| |d	dgd�}t|d
d	gddggƒ t | ¡d }t| |d	|gd�}t|d
dgddggƒ d S )NFr‘   r   r7   rÁ   rü   r™   rª   r2   r   rQ   )rO   r   r,   r<   Úmax)rL   rK   ri   rê   Zextra_labelrM   rM   rN   Ú.test_confusion_matrix_multiclass_subset_labelsc  s    r�  zlabels, err_msgz,'labels' should contains at least one label.rª   r™   z.At least one label specified must be in y_truez
empty listzunknown labels)Zidsc                 C   sP   t dd�\}}}tjt|d�� t||| d� W d   ƒ n1 sB0    Y  d S )NFr‘   rÏ   rÁ   )rO   r§   rº   r»   r   )ra   rÔ   rL   rK   ri   rM   rM   rN   Útest_confusion_matrix_errorv  s    	r‚  ra   )ÚNonerD   Ú
multiclassc                 C   s>   | rt | ƒnd}tj||ftd�}tg g | d�}t||ƒ d S )Nr   ©r  rÁ   )rd   r<   rŒ   r?   r   r,   )ra   Zexpected_n_classesÚexpectedrê   rM   rM   rN   Ú*test_confusion_matrix_on_zero_length_input„  s    r‡  c                  C   s<  g d¢} t  t| ƒ¡}t| | ƒ}|jt jks0J ‚t jt jt jfD ],}t| | |j	|dd�d�}|jt jks@J ‚q@t j
t jd tfD ],}t| | |j	|dd�d�}|jt jks~J ‚q~t jt| ƒdt jd�}t| | |d�}|d dksàJ ‚|d d	ksðJ ‚t jt| ƒd
t jd�}t| | |d�}|d d
k�s&J ‚|d dk�s8J ‚d S )Nr†   F)Úcopyr  l   ÿÿ r…  ©r   r   rl  l   þÿ l   ÿÿÿÿ éþÿÿÿ)r<   Zonesrd   r   r  Úint64Zbool_Zint32Zuint64ZastypeÚfloat32Úfloat64ÚobjectÚfullZuint32)rF   Úweightrê   r  rM   rM   rN   Útest_confusion_matrix_dtypeŽ  s$    
r‘  r  )ZInt64ZFloat64Úbooleanc                 C   sZ   t  d¡}t g d¢¡}|j|| d�}|jg d¢dd�}t||ƒ}t||ƒ}t||ƒ dS )zkChecks that confusion_matrix works with pandas nullable dtypes.

    Non-regression test for gh-25635.
    Úpandas)	r7   r   r   r7   r   r7   r7   r   r7   r…  )	r   r   r7   r7   r   r7   r7   r7   r7   r‹  N)r§   Zimportorskipr<   rŠ   ÚSeriesr   r,   )r  ÚpdZ	y_ndarrayrL   Zy_predictedÚoutputZexpected_outputrM   rM   rN   Ú%test_confusion_matrix_pandas_nullable©  s    


r—  c                  C   sL   t  ¡ } t| dd�\}}}d}t||t t| jƒ¡| jd�}||ksHJ ‚d S )NFrP   a|                precision    recall  f1-score   support

      setosa       0.83      0.79      0.81        24
  versicolor       0.33      0.10      0.15        31
   virginica       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
©ra   rb   ©r	   r8   rO   r   r<   r=   rd   rb   ©rh   rL   rK   ri   rj   rk   rM   rM   rN   Ú%test_classification_report_multiclass»  s    ür›  c                  C   s0   g d¢g d¢ } }d}t | |ƒ}||ks,J ‚d S )N)	r   r   r   r7   r7   r7   r2   r2   r2   rd  a|                precision    recall  f1-score   support

           0       0.33      0.33      0.33         3
           1       0.33      0.33      0.33         3
           2       0.33      0.33      0.33         3

    accuracy                           0.33         9
   macro avg       0.33      0.33      0.33         9
weighted avg       0.33      0.33      0.33         9
r„   )rL   rK   rj   rk   rM   rM   rN   Ú.test_classification_report_multiclass_balancedÕ  s    
rœ  c                  C   s8   t  ¡ } t| dd�\}}}d}t||ƒ}||ks4J ‚d S )NFrP   a|                precision    recall  f1-score   support

           0       0.83      0.79      0.81        24
           1       0.33      0.10      0.15        31
           2       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
)r	   r8   rO   r   rš  rM   rM   rN   Ú:test_classification_report_multiclass_with_label_detectionç  s
    
r�  c                  C   sN   t  ¡ } t| dd�\}}}d}t||t t| jƒ¡| jdd�}||ksJJ ‚d S )NFrP   a|                precision    recall  f1-score   support

      setosa    0.82609   0.79167   0.80851        24
  versicolor    0.33333   0.09677   0.15000        31
   virginica    0.41860   0.90000   0.57143        20

    accuracy                        0.53333        75
   macro avg    0.52601   0.59615   0.50998        75
weighted avg    0.51375   0.53333   0.47310        75
r˜   )ra   rb   Údigitsr™  rš  rM   rM   rN   Ú1test_classification_report_multiclass_with_digitsû  s    ûrŸ  c                  C   st   t dd�\} }}t g d¢¡|  } t g d¢¡| }d}t| |ƒ}||ksNJ ‚d}t| |g d¢d�}||kspJ ‚d S )NFr‘   )ÚblueÚgreenÚreda|                precision    recall  f1-score   support

        blue       0.83      0.79      0.81        24
       green       0.33      0.10      0.15        31
         red       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
a|                precision    recall  f1-score   support

           a       0.83      0.79      0.81        24
           b       0.33      0.10      0.15        31
           c       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
ru   ©rb   ©rO   r<   rŠ   r   )rL   rK   ri   rj   rk   rM   rM   rN   Ú7test_classification_report_multiclass_with_string_label  s    
r¥  c                  C   sL   t dd�\} }}t g d¢¡}||  } || }d}t| |ƒ}||ksHJ ‚d S )NFr‘   )u   blueÂ¢u   greenÂ¢u   redÂ¢u                precision    recall  f1-score   support

       blueÂ¢       0.83      0.79      0.81        24
      greenÂ¢       0.33      0.10      0.15        31
        redÂ¢       0.42      0.90      0.57        20

    accuracy                           0.53        75
   macro avg       0.53      0.60      0.51        75
weighted avg       0.51      0.53      0.47        75
r¤  ©rL   rK   ri   ra   rj   rk   rM   rM   rN   Ú8test_classification_report_multiclass_with_unicode_label9  s    
r§  c                  C   sL   t dd�\} }}t g d¢¡}||  } || }d}t| |ƒ}||ksHJ ‚d S )NFr‘   )r   Zgreengreengreengreengreenr¢  a×                             precision    recall  f1-score   support

                     blue       0.83      0.79      0.81        24
greengreengreengreengreen       0.33      0.10      0.15        31
                      red       0.42      0.90      0.57        20

                 accuracy                           0.53        75
                macro avg       0.53      0.60      0.51        75
             weighted avg       0.51      0.53      0.47        75
r¤  r¦  rM   rM   rN   Ú<test_classification_report_multiclass_with_long_string_labelO  s    
r¨  c                  C   sb   g d¢} g d¢}g d¢}d}t jt|d��$ t| |ddg|d� W d   ƒ n1 sT0    Y  d S )	N©r   r   r2   r   r   ©r   r2   r2   r   r   ©zclass 0zclass 1zclass 2z6labels size, 2, does not match size of target_names, 3rÏ   r   r2   r˜  )r§   rÞ   rß   r   )rL   rK   rb   r�   rM   rM   rN   Ú=test_classification_report_labels_target_names_unequal_lengthf  s    r¬  c                  C   s\   g d¢} g d¢}g d¢}d}t jt|d�� t| ||d� W d   ƒ n1 sN0    Y  d S )Nr©  rª  r«  zaNumber of classes, 2, does not match size of target_names, 3. Try specifying the labels parameterrÏ   r£  )r§   rº   r»   r   )rL   rK   rb   rÔ   rM   rM   rN   Ú@test_classification_report_no_labels_target_names_unequal_lengthp  s    ÿr­  c                  C   sN   d} d}t d|| dd�\}}t d|| dd�\}}d}t||ƒ}||ksJJ ‚d S )Nr™   r6  r7   r   )rH   rG   rh  r6   aè                precision    recall  f1-score   support

           0       0.50      0.67      0.57        24
           1       0.51      0.74      0.61        27
           2       0.29      0.08      0.12        26
           3       0.52      0.56      0.54        27

   micro avg       0.50      0.51      0.50       104
   macro avg       0.45      0.51      0.46       104
weighted avg       0.45      0.51      0.46       104
 samples avg       0.46      0.42      0.40       104
)r   r   )rh  rG   ri   rL   rK   rj   rk   rM   rM   rN   Ú%test_multilabel_classification_report~  s    ÿ
ÿ

r®  c                  C   sÊ   t  g d¢g d¢g¡} t  g d¢g d¢g¡}t| |ƒdks>J ‚t| | ƒdksPJ ‚t||ƒdksbJ ‚t|t  |¡ƒdkszJ ‚t| t  | ¡ƒdks’J ‚t| t  | j¡ƒdks¬J ‚t|t  | j¡ƒdksÆJ ‚d S )Nr†   r‡   rˆ   r‰   r   r7   )r<   rŠ   r!   r‹   rŒ   r;   r�   rM   rM   rN   Ú$test_multilabel_zero_one_loss_subset�  s    r¯  c                  C   sN  t  g d¢g d¢g¡} t  g d¢g d¢g¡}t  ddg¡}t| |ƒdksLJ ‚t| | ƒdks^J ‚t||ƒdkspJ ‚t|d| ƒdks†J ‚t| d|  ƒdksœJ ‚t| t  | j¡ƒdks¶J ‚t|t  | j¡ƒd	ksÐJ ‚t| ||d
�dksæJ ‚t| d| |d
�dk�sJ ‚t| t  | ¡|d
�dk�s J ‚t| d |d ƒt| d |d ƒk�sJJ ‚d S )Nr†   r‡   rˆ   r7   rª   çUUUUUUÅ?r   rÂ   r‰   r  gUUUUUUµ?gUUUUUUí?)r<   rŠ   r   rŒ   r;   Ú
zeros_likeÚ
sp_hamming)rŽ   r�   ÚwrM   rM   rN   Útest_multilabel_hamming_loss«  s    r´  c                  C   s´  t  g d¢¡} t  g d¢¡}d}tjt|d��  t| |ddd� W d   ƒ n1 sT0    Y  t  g d¢g d¢g¡} t  g d	¢g d
¢g¡}d}tjt|d��  t| |ddd� W d   ƒ n1 sÂ0    Y  t  g d¢¡} t  g d¢¡}d}tjt|d�� t| |dd� W d   ƒ n1 �s 0    Y  d}tjt|d�� t| |dd� W d   ƒ n1 �sb0    Y  d}tjt|d��  t| |ddd� W d   ƒ n1 �s¦0    Y  d S )N)r   r7   r   r7   r7   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]rÏ   rD   r2   ©r“   rÒ   r†   r³   r´   r‡   ú•Target is multilabel-indicator but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted', 'samples'\].r¥   )r   r7   r7   r   r2   re  ú€Target is multiclass but average='binary'. Please choose another average setting, one of \[None, 'micro', 'macro', 'weighted'\].r’   zJSamplewise metrics are not available outside of multilabel classification.r±   zšNote that pos_label \(set to 3\) is ignored when average != 'binary' \(got 'micro'\). You may use labels=\[pos_label\] to specify a single positive class.r¯   rª   )r<   rŠ   r§   rº   r»   r   rÞ   rß   )rL   rK   rÔ   Zmsg1Úmsg2Úmsg3r�   rM   rM   rN   Útest_jaccard_score_validation¿  s0    .ÿ.ÿ..ÿrº  c                 C   sb  t  g d¢g d¢g¡}t  g d¢g d¢g¡}t||dd�dksBJ ‚t||dd�dksXJ ‚t||dd�dksnJ ‚t|t  |¡dd�dksŠJ ‚t|t  |¡dd�dks¦J ‚t|t  |j¡dd�dksÄJ ‚t|t  |j¡dd�dksâJ ‚t  g d¢g d	¢g¡}t  g d
¢g d¢g¡}tt||dd�dƒ tt||dd�dƒ tt||dd�dƒ tt||dddgd�dƒ tt||dddgd�dƒ tt||d d�t  g d¢¡ƒ t  g d¢g d¢g¡}t  g d
¢g d¢g¡}tt||dd�dƒ tt||dd�dƒ d}tj	t
|d��" t||dgdd� W d   ƒ n1 �s,0    Y  d}tj	t
|d��" t||dgdd� W d   ƒ n1 �sr0    Y  d}tjt|d��@ tt  ddgg¡t  ddgg¡dd�dk�sÀJ ‚W d   ƒ n1 �sÖ0    Y  d}tjt|d��L tt  ddgddgg¡t  ddgddgg¡dd�dk�s0J ‚W d   ƒ n1 �sF0    Y  t| ƒ�r^J ‚d S )Nr†   r‡   rˆ   r±   r’   r¶   r7   r   r³   r´   r®   rÂ   r¯   r×   g«ªªªªªâ?r2   rµ   r‰   )r‰   r£   r‰   r·   r°   g      ì?z	Got 4 > 2rÏ   r™   r­   z
Got -1 < 0r¥   zXJaccard is ill-defined and being set to 0.0 in labels with no true or predicted samples.zXJaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels.)r<   rŠ   r   r‹   rŒ   r;   r*   r,   r§   rº   r»   rÞ   r   r  )ÚrecwarnrŽ   r�   rL   rK   r¸  r¹  r�   rM   rM   rN   Útest_multilabel_jaccard_scoreç  sh    ÿÿÿ22ÿ$ÿÿ(ÿýûÿ(	r¼  c                 C   s^  g d¢}g d¢}g d¢}t ƒ }| |¡ | |¡}| |¡}tt||ƒ}tt||ƒ}ddgddgddgdgdgdgd g}	ddgdd	gd	dgdgdgd	gd g}
d
D ]2}t|	|
ƒD ]"\}}t|||d�|||d�ƒ q®q t ddgddgddgg¡}t ddgddgddgg¡}t	ƒ �( t||dd�dk�s,J ‚W d   ƒ n1 �sB0    Y  t
| ƒ�rZJ ‚d S )N)Úantr½  Úcatr¾  r½  r¾  Úbirdr¿  )r¾  r½  r¾  r¾  r½  r¿  r¿  r¾  )r½  r¿  r¾  r½  r¿  r¾  r   r7   r2   )r®   r°   r¯   Nrµ   r°   r’   )r%   rB   Z	transformr   r   Úzipr*   r<   rŠ   r-   r  )r»  rL   rK   ra   Zlbr¼   r½   Zmulti_jaccard_scoreZbin_jaccard_scoreZmulti_labels_listZbin_labels_listr“   Zm_labelZb_labelrM   rM   rN   Útest_multiclass_jaccard_score4  s:    


ù	$

þ8rÁ  c                 C   sÜ   t dgdgdd�dksJ ‚d}tjt|d��. t ddgddgdd�dksLJ ‚W d   ƒ n1 s`0    Y  t dgdgddd�d	ks†J ‚t g d
¢¡}t g d¢¡}tt ||dd�dƒ tt ||ddd�dƒ t| ƒrØJ ‚d S )Nr7   r   rD   r’   ro   zOJaccard is ill-defined and being set to 0.0 due to no true or predicted samplesrÏ   rÝ   r£   )r7   r   r7   r7   r   )r7   r   r7   r7   r7   r¶   rµ  r‰   )r   r§   rÞ   r   r<   rŠ   r*   r  )r»  r�   rL   rK   rM   rM   rN   Ú!test_average_binary_jaccard_scoreY  s    ÿ<ÿrÂ  c                  C   s„   t  g d¢g d¢g¡} t  g d¢g d¢g¡}d}tjt|d��2 t| |ddd�}|t d¡ksbJ ‚W d   ƒ n1 sv0    Y  d S )	Nr‡   ©r   r   r   z�Jaccard is ill-defined and being set to 0.0 in samples with no true or predicted labels. Use `zero_division` parameter to control this behavior.rÏ   r±   rt   ©r“   rs   ro   )r<   rŠ   r§   rÞ   r   r   r¨   )rL   rK   r�   ÚscorerM   rM   rN   Ú(test_jaccard_score_zero_division_warningp  s    ÿrÆ  zzero_division, expected_scorer‰  )r7   r‰   c                 C   s†   t  g d¢g d¢g¡}t  g d¢g d¢g¡}t ¡ �, t dt¡ t||d| d�}W d   ƒ n1 sf0    Y  |t |¡ks‚J ‚d S )Nr‡   rÃ  r–   r±   rÄ  )	r<   rŠ   r|   r}   rš   r   r   r§   r¨   )rs   Úexpected_scorerL   rK   rÅ  rM   rM   rN   Ú*test_jaccard_score_zero_division_set_value  s    
ÿ$rÈ  c                  C   s8  t  g d¢g d¢g d¢g¡} t  g d¢g d¢g d¢g¡}t| |d d�\}}}}t|g d¢dƒ t|g d¢dƒ t|g d	¢dƒ t|g d
¢dƒ t| |dd d�}|}t|g d¢dƒ t| |dd�\}}}}t|dƒ t|dƒ t|dƒ |d u sòJ ‚tt| |ddd�t  |¡ƒ t| |dd�\}}}}t|dƒ t|dƒ t|dƒ |d u �sPJ ‚tt| |ddd�d| | d| |  ƒ t| |dd�\}}}}t|dƒ t|dƒ t|dƒ |d u �s¼J ‚tt| |ddd�t j||d�ƒ t| |dd�\}}}}t|dƒ t|dƒ t|dƒ |d u �sJ ‚tt| |ddd�dƒ d S )N©r7   r   r   r   ©r   r7   r   r   ry  )r7   r   r7   r   r’   )ro   r‰   r£   ro   r2   )ro   r£   r£   ro   )ro   rÂ   r7   ro   )r7   r7   r7   r7   ©r—   r“   )r   rr  r7   r   r®   g      Ø?r‰   g«ªªªªªÚ?r¯   r˜   r™   r°   r<  r±   ©r<   rŠ   r   r+   r   r*   r¹   r“   ©rL   rK   rI   r›   rœ   r�   Úf2rU   rM   rM   rN   Ú+test_precision_recall_f1_score_multilabel_1Œ  sT    


ÿ


þ


þ


rÏ  c                  C   s:  t  g d¢g d¢g d¢g¡} t  g d¢g d¢g d¢g¡}t| |d d�\}}}}t|g d¢dƒ t|g d	¢dƒ t|g d
¢dƒ t|g d¢dƒ t| |dd d�}|}t|g d¢dƒ t| |dd�\}}}}t|dƒ t|dƒ t|dƒ |d u sòJ ‚tt| |ddd�d| | d| |  ƒ t| |dd�\}}}}t|dƒ t|dƒ t|dƒ |d u �s^J ‚tt| |ddd�t  |¡ƒ t| |dd�\}}}}t|dƒ t|dƒ t|dƒ |d u �s¼J ‚tt| |ddd�t j||d�ƒ t| |dd�\}}}}t|dƒ t|dƒ t|dƒ |d u �sJ ‚tt| |ddd�ddƒ d S )NrÉ  rÊ  r{  ©r   r   r   r7   rf  r’   )ro   r£   ro   ro   r2   )ro   r‰   ro   ro   )ro   g…ëQ¸å?ro   ro   ©r7   r2   r7   r   rË  )r   çš™™™™™á?r   r   r¯   ç      Ð?r˜   r™   r®   g      À?r°  r°   r‰   rV   r<  r±   g¥½Á&SÅ?rÌ  rÍ  rM   rM   rN   Ú+test_precision_recall_f1_score_multilabel_2Ï  sX    


þ


ÿ


þ


ÿrÔ  z%zero_division, zero_division_expected)rt   r   rl  c                 C   s²  t  g d¢g d¢g d¢g¡}t  g d¢g d¢g d¢g¡}t||d | d�\}}}}t||dddgd	ƒ t|dd
d|gd	ƒ d}t||dd|gd	ƒ t|g d¢d	ƒ t||d	d | d�}	|}
t|	|dd|gd	ƒ t||d| d�\}}}}t  |¡rðdn|}dt  |¡  }t|d	| | ƒ t|d| | ƒ d}t||ƒ |d u �sDJ ‚tt||d	d| d�t|	d d�ƒ t||d| d�\}}}}t|dƒ t|d
ƒ t|dƒ |d u �s¨J ‚tt||d	d| d�d| | d| |  ƒ t||d| d�\}}}}t||dk�rþdndƒ t|d
ƒ d}t|d| ƒ |d u �s.J ‚tt||d	d| d�t|	|
d�ƒ t||dd�\}}}}t|dƒ t|dƒ t|dƒ |d u �s�J ‚d }tt||d	d| d�|d	ƒ d S )!NrÊ  rÉ  r{  r%  rÐ  rÄ  r£   ro   r2   r‰   r   rÂ   r7   rÑ  ©r—   r“   rs   rÒ  r®   rª   ç      ø?gªªªªªªÚ?r<  r¯   rY   r˜   r™   r°   r¶   gªªªªªª@r±   r’   rV   gZd;ßOÕ?)r<   rŠ   r   r+   r   rB  r*   r.   )rs   Zzero_division_expectedrL   rK   rI   r›   rœ   r�   Z
expected_frÎ  rU   Zvalue_to_sumZvalues_to_averageZexpected_resultrM   rM   rN   Ú7test_precision_recall_f1_score_with_an_empty_prediction  s’    ÿÿ
û
øÿ



ÿüÿ

ÿ
ü



ÿûr×  r—   )r®   r¯   r°   r±   c                 C   sä   t  d¡}t  |¡}t ¡ �F t d¡ t|||| |d�\}}}}t||| ||d�}	W d   ƒ n1 sh0    Y  |d u s~J ‚t  |¡r¬||||	fD ]}
t  |
¡s”J ‚q”d S t	|ƒ}t
||ƒ t
||ƒ t
||ƒ t
|	t	|ƒƒ d S )N©rZ   rª   r–   ©r“   r—   rs   rÕ  )r<   rŒ   r±  r|   r}   rš   r   r   rB  rg   r*   )r—   r“   rs   rL   rK   rI   r›   rœ   r�   Úfbetarm   rM   rM   rN   Ú"test_precision_recall_f1_no_labelss  s8    



ûû$



rÛ  c           	      C   sÌ   t  d¡}t  |¡}t}t t¡�( |||| dd�\}}}}W d   ƒ n1 sP0    Y  t|dƒ t|dƒ t|dƒ |d u s„J ‚t t¡�  t||| dd�}W d   ƒ n1 s´0    Y  t|dƒ d S )NrØ  r£   rw  r   )	r<   rŒ   r±  r   r§   rÞ   r   r*   r   )	r“   rL   rK   ÚfuncrI   r›   rœ   r�   rÚ  rM   rM   rN   Ú1test_precision_recall_f1_no_labels_check_warnings›  s    

6


.rÝ  c                 C   sØ   t  d¡}t  |¡}t ¡ �F t d¡ t||d d| d�\}}}}t||dd | d�}W d   ƒ n1 sh0    Y  t  | ¡} t	|| | | gdƒ t	|| | | gdƒ t	|| | | gdƒ t	|g d¢dƒ t	|| | | gdƒ d S )NrØ  r–   r£   rÙ  rÕ  r2   rÃ  )
r<   rŒ   r±  r|   r}   rš   r   r   r�  r+   )rs   rL   rK   rI   r›   rœ   r�   rÚ  rM   rM   rN   Ú/test_precision_recall_f1_no_labels_average_none¯  s(    




û
ÿ$
rÞ  c                  C   sä   t  d¡} t  | ¡}t t¡�( t| |d dd�\}}}}W d   ƒ n1 sL0    Y  t|g d¢dƒ t|g d¢dƒ t|g d¢dƒ t|g d¢dƒ t t¡�  t| |dd d�}W d   ƒ n1 sÆ0    Y  t|g d¢dƒ d S )NrØ  r7   rw  rÃ  r2   rË  )	r<   rŒ   r±  r§   rÞ   r   r   r+   r   )rL   rK   rI   r›   rœ   r�   rÚ  rM   rM   rN   Ú4test_precision_recall_f1_no_labels_average_none_warnÓ  s    


ÿ,.rß  c               	   C   sv  t t } }dD ]”}d}tj||d��& | g d¢g d¢|d� W d   ƒ n1 sP0    Y  d}tj||d��& | g d¢g d¢|d� W d   ƒ q1 s˜0    Y  qd}tj||d��B | t d	d
gd	d
gg¡t d	d
gd
d
gg¡dd� W d   ƒ n1 �s 0    Y  d}tj||d��B | t d	d
gd
d
gg¡t d	d
gd	d
gg¡dd� W d   ƒ n1 �sf0    Y  d}tj||d��B | t d	d	gd	d	gg¡t d
d
gd
d
gg¡dd� W d   ƒ n1 �sÌ0    Y  d}tj||d��B | t d
d
gd
d
gg¡t d	d	gd	d	gg¡dd� W d   ƒ n1 �s20    Y  d}tj||d��& | d	d	gddgdd� W d   ƒ n1 �s|0    Y  d}tj||d��& | ddgd	d	gdd� W d   ƒ n1 �sÆ0    Y  tjdd��„}t d¡ t d
d
gd
d
gdd� d}t	| 
¡ jƒ|k�sJ ‚d}t	| 
¡ jƒ|k�s6J ‚d}t	| 
¡ jƒ|k�sRJ ‚W d   ƒ n1 �sh0    Y  d S )N©Nr°   r®   zŠPrecision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.rÏ   rƒ   ©r7   r7   r2   r’   z‚Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.zŠPrecision is ill-defined and being set to 0.0 in samples with no predicted labels. Use `zero_division` parameter to control this behavior.r7   r   r±   z‚Recall is ill-defined and being set to 0.0 in samples with no true labels. Use `zero_division` parameter to control this behavior.ú‚Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.r¯   úzRecall is ill-defined and being set to 0.0 due to no true samples. Use `zero_division` parameter to control this behavior.r¥   rD   Trz   Úalwaysú‰F-score is ill-defined and being set to 0.0 due to no true nor predicted samples. Use `zero_division` parameter to control this behavior.)r   r   r§   rÞ   r<   rŠ   r|   r}   rš   r~   Úpopr   )rœ   r³  r“   r�   r{   rM   rM   rN   Útest_prf_warningsï  s\    
ÿ4ÿ6ÿRÿRÿRÿRÿ6ÿ6
ÿÿÿrç  c                 C   sÖ  t  ¡ ��T t  d¡ dD ]4}tg d¢g d¢|| d� tg d¢g d¢|| d� qtt ddgddgg¡t ddgddgg¡d| d� tt ddgddgg¡t ddgddgg¡d| d� tt ddgddgg¡t ddgddgg¡d	| d� tt ddgddgg¡t ddgddgg¡d	| d� tddgd
d
gd| d� td
d
gddgd| d� W d   ƒ n1 �sf0    Y  t jdd��D}t  d¡ tddgddgd| d� t|ƒdk�s²J ‚W d   ƒ n1 �sÈ0    Y  d S )Nr–   rà  rƒ   rá  rÄ  r7   r   r±   r¯   r¥   rD   Trz   rä  )r|   r}   rš   r   r<   rŠ   rd   )rs   r“   r{   rM   rM   rN   Ú)test_prf_no_warnings_if_zero_division_setW  sZ    
ÿÿüüüüÿÿ&
ÿrè  c                 C   s:  t  ¡ �N t  d¡ tt ddgddgg¡t ddgddgg¡d| d� W d   ƒ n1 s\0    Y  t jdd��²}t  d¡ tt ddgddgg¡t ddgddgg¡d| d� | d	krÒt| ¡ jƒd
ksâJ ‚nt	|ƒdksâJ ‚tddgddgƒ | d	k�rt| ¡ jƒd
k�sJ ‚W d   ƒ n1 �s,0    Y  d S )Nr–   r7   r   r¯   rÄ  Trz   rä  rt   rã  )
r|   r}   rš   r    r<   rŠ   r~   ræ  r   rd   ©rs   r{   rM   rM   rN   Útest_recall_warnings•  s0    

ü$
üÿ
ÿrê  c                 C   s6  t jdd��®}t  d¡ tt ddgddgg¡t ddgddgg¡d| d� | dkrlt| ¡ jƒd	ks|J ‚nt	|ƒdks|J ‚tddgddgƒ | dkr¬t| ¡ jƒd	ks¬J ‚W d   ƒ n1 sÀ0    Y  t  ¡ �N t  d
¡ tt ddgddgg¡t ddgddgg¡d| d� W d   ƒ n1 �s(0    Y  d S )NTrz   rä  r7   r   r¯   rÄ  rt   râ  r–   )
r|   r}   rš   r   r<   rŠ   r~   ræ  r   rd   ré  rM   rM   rN   Útest_precision_warnings½  s0    
üÿÿ$

ürë  c              	   C   sD  t jdd��� }t  d¡ tttdd�fD ]ô}|t ddgddgg¡t ddgddgg¡d| d	� t|ƒdksrJ ‚|t ddgddgg¡t ddgddgg¡d| d	� t|ƒdks¶J ‚|t ddgddgg¡t ddgddgg¡d| d	� | d
k�rt	| 
¡ jƒdk�sJ ‚q*t|ƒdks*J ‚q*W d   ƒ n1 �s60    Y  d S )NTrz   rä  r2   r¤   r7   r   r¯   rÄ  rt   rå  )r|   r}   rš   r   r   r   r<   rŠ   rd   r~   ræ  r   )rs   r{   rÅ  rM   rM   rN   Útest_fscore_warningså  s6    
üüü
ÿ
rì  c            
   
   C   sÆ   g d¢} g d¢}d}t  g d¢g d¢g d¢g¡}t  g d¢g d¢g d¢g¡}d}| ||f|||ffD ]\\}}}tttttd	d
�fD ]<}	tjt	|d�� |	||ƒ W d   ƒ q‚1 s´0    Y  q‚qdd S )N)r7   r2   rª   rª   )r7   r2   rª   r7   r·  r†   r³   rˆ   r
  r¶  r2   r¤   rÏ   )
r<   rŠ   r   r    r   r   r   r§   rº   r»   )
Z	y_true_mcZ	y_pred_mcZmsg_mcZ
y_true_indZ
y_pred_indZmsg_indrL   rK   r�   rm   rM   rM   rN   Ú'test_prf_average_binary_data_non_binary	  s$    ÿÿþ
ürí  c               *   C   sÞ  d} d}d}d}d}d}| t  g d¢g d¢g d	¢g¡f| t  d
dgdd
gddgg¡f|g d¢f|g d¢f|g d¢f|t  dgdgdgg¡f|t  d
gdgdgg¡f|t  dgdgdgg¡f|t  d
dgddgddgg¡f|t  ddgddgddgg¡fg
}| | f| ||f|||f||| fd || fd ||f|||fd ||fd ||fd | |fd ||fd ||fd ||fd ||fd | |fd ||fd ||fd ||fd | |fd ||fd ||fd i}t|dd�D �]Ö\\}}	\}
}z|||
f }W n  t�yô   ||
|f }Y n0 |d u �rÞt t¡� t|	|ƒ W d   ƒ n1 �s,0    Y  ||
k�rˆd ||
¡}tjt|d�� t|	|ƒ W d   ƒ n1 �s|0    Y  nT|||| fv�rˆd |¡}tjt|d�� t|	|ƒ W d   ƒ n1 �sÒ0    Y  nªt|	|ƒ\}}}||k�süJ ‚| 	d¡�r*|jdk�sJ ‚|jdk�sJJ ‚n t
|t  |	¡ƒ t
|t  |¡ƒ t t¡�" t|	d d… |ƒ W d   ƒ n1 �s~0    Y  �q²ddg}	d d!g}d"}tjt|d�� t|	|ƒ W d   ƒ n1 �sÐ0    Y  d S )#Nzmultilabel-indicatorr„  rD   Z
continuouszmulticlass-multioutputzcontinuous-multioutputr†   r³   rˆ   r   r7   )r2   rª   r7   )ro   rÖ  r£   r2   rª   ro   rÖ  r£   r‰   r!  gš™™™™™ñ?g      @)rp  z@Classification metrics can't handle a mix of {0} and {1} targetsrÏ   z{0} is not supportedZ
multilabelZcsrr¥   )r7   r2   )r   r2   rª   )r2   )r   r2   zÝYou appear to be using a legacy multi-label data representation. Sequence of sequences are no longer supported; use a binary array or sparse matrix instead - the MultiLabelBinarizer transformer can convert to this format.)r<   rŠ   r   ÚKeyErrorr§   rº   r»   r"   ÚformatÚ
startswithr,   Zsqueeze)ZINDZMCZBINZCNTZMMCZMCNZEXAMPLESZEXPECTEDZtype1rŽ   Ztype2r�   r†  rÔ   Zmerged_typeZy1outZy2outr�   rM   rM   rN   Útest__check_targets+	  s’    


õê
*
ÿÿ,
,6ÿrñ  c                  C   s*   ddg} ddg}t | |ƒd dks&J ‚d S )Nr   r7   r¥   r„  )r"   r'  rM   rM   rN   ÚAtest__check_targets_multiclass_with_both_y_true_and_y_pred_binary�	  s    rò  c                  C   s`   t  g d¢¡} t  g d¢¡}t| |ƒdks.J ‚t  g d¢¡} t  g d¢¡}t| |ƒdks\J ‚d S )N)r¥   r7   r7   r¥   )g      !Àr‰   rÖ  g333333Ó¿rÌ   )r   r2   r2   r   )r<   rŠ   r   ©rL   Úpred_decisionrM   rM   rN   Útest_hinge_loss_binary”	  s    rõ  c               
   C   s  t  g d¢g d¢g d¢g d¢g d¢g d¢g¡} t  g d¢¡}t  d| d d  | d d  d| d d  | d d	  d| d	 d	  | d	 d
  d| d
 d  | d
 d	  d| d d
  | d d	  d| d d	  | d d
  g¡}t j|dd |d� t  |¡}t|| ƒ|k�sJ ‚d S )N©ç
×£p=
×?çÃõ(\�ÂÅ¿ç�Âõ(\�â¿g®Gáz®ï¿)çHáz®Gá¿g®Gáz®×¿ç¸…ëQ¸Þ¿rù  ©ç333333÷¿rù  çR¸…ëQØ¿rø  )rú  rþ  rû  rù  ©gáz®GáÀgHáz®Gé¿gHáz®GÑ¿g¸…ëQ¸Î?)r   r7   r2   r7   rª   r2   r7   r   r2   rª   r™   r˜   ©Úout©r<   rŠ   Úclipr¹   r   )rô  rL   Údummy_lossesÚdummy_hinge_lossrM   rM   rN   Útest_hinge_loss_multiclassž	  s,    úÿ
úÿ

r  c                  C   sp   t  g d¢¡} t  g d¢g d¢g d¢g d¢g¡}d}tjt|d�� t| |ƒ W d   ƒ n1 sb0    Y  d S )N)r   r7   r2   r2   )gR¸…ëQô?gœÄ °rh¡?gÃõ(\�Âå¿gffffffö¿rü  rÿ  zDPlease include all labels in y_true or pass labels as third argumentrÏ   )r<   rŠ   r§   rº   r»   r   )rL   rô  Úerror_messagerM   rM   rN   Ú:test_hinge_loss_multiclass_missing_labels_with_labels_none¹	  s    üÿ	ÿr  c               
   C   sæ   t  g d¢¡} t  g d¢¡}d}tjtt |¡d�� t| |d� W d   ƒ n1 sV0    Y  t  ddgddgddgddgddgddgddgg¡}g d	¢}d
}tjtt |¡d�� t| ||d� W d   ƒ n1 sØ0    Y  d S )N)r2   r7   r   r7   r   r7   r7   )r   r7   r2   r7   r   r2   r7   z”The shape of pred_decision cannot be 1d arraywith a multiclass target. pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7,)rÏ   ró  r   r7   r2   rƒ   z²The shape of pred_decision is not consistent with the number of classes. With a multiclass target, pred_decision shape must be (n_samples, n_classes), that is (7, 3). Got: (7, 2))rL   rô  ra   )r<   rŠ   r§   rº   r»   ÚreÚescaper   )rL   rô  r  ra   rM   rM   rN   Ú<test_hinge_loss_multiclass_no_consistent_pred_decision_shapeÊ	  s    ÿ*4ÿr  c               	   C   s  t  g d¢g d¢g d¢g d¢g d¢g¡} t  g d¢¡}t  g d¢¡}t  d| d d  | d d  d| d d  | d d  d| d d  | d d	  d| d	 d  | d	 d  d| d
 d  | d
 d	  g¡}t j|dd |d� t  |¡}t|| |d�|k�sJ ‚d S )Nrö  ©çš™™™™™á¿rþ  rû  rù  rü  )r   r7   r2   r7   r2   )r   r7   r2   rª   r7   r   r2   rª   r™   r   rÁ   r  ©rô  rL   ra   r  r  rM   rM   rN   Ú.test_hinge_loss_multiclass_with_missing_labelsæ	  s*    ûÿ	ûÿ	
r  c               	   C   s  t  g d¢g d¢g d¢g d¢g d¢g¡} t  g d¢¡}t  g d¢¡}t  d| d d  | d d  d| d d	  | d d  d| d	 d	  | d	 d  d| d
 d  | d
 d	  d| d d	  | d d  g¡}t j|dd |d� t  |¡}tt|| |d�|ƒ d S )N)r÷  rø  rù  )g333333Ã¿rù  rû  )rý  rù  rþ  )r  gö(\�Âõè¿gáz®GáÚ¿)r   r2   r2   r   r2   rƒ   r7   r   r2   rª   r™   r   rÁ   )r<   rŠ   r  r¹   r*   r   r  rM   rM   rN   Ú@test_hinge_loss_multiclass_missing_labels_only_two_unq_in_y_true 
  s.    ûÿ	ûÿ	
ÿr  c               
   C   s
  g d¢} g d¢g d¢g d¢g d¢g d¢g d¢g}t  d|d d  |d d  d|d d  |d d  d|d d  |d d	  d|d	 d  |d	 d  d|d
 d	  |d
 d  d|d d  |d d	  g¡}t j|dd |d� t  |¡}t| |ƒ|k�sJ ‚d S )N)r   r¡  r¢  r¡  Úwhiter¢  rö  r  rü  rÿ  r7   r   r2   rª   r™   r˜   r   r  )rL   rô  r  r  rM   rM   rN   Ú+test_hinge_loss_multiclass_invariance_lists 
  s(    úúÿ

r  c            	   	   C   sž  g d¢} t  ddgddgddgddgddgd	d
gg¡}t| |ƒ}t  t t  | ¡dk|d d …df ¡¡ }t||ƒ g d¢} g d¢g d¢g d¢g}t| |dd�}t|dƒ | d9 } |d9 }t| |dd�}t|dƒ g d¢} ddgddgddgg}t t	¡� t| |ƒ W d   ƒ n1 �s0    Y  g d¢} ddgddgddgddgg}t| |ƒ}t|dƒ ddg} ddgddgg}t  ddgddgg¡}d}tjt	|d �� t| |ƒ W d   ƒ n1 �sÀ0    Y  ddgddgddgg}d!}tjt	|d �� t| |ƒ W d   ƒ n1 �s0    Y  t  t  
|d d …df ¡¡ }t| |ddgd"�}t||ƒ g d#¢} g d$¢g d¢g d%¢g}t| |g d&¢d"�}t|t  
d¡ ƒ d S )'N©Únor  r  Úyesr  r  r‰   rÉ   rX   ç{®Gáz„?ç®Gáz®ï?r¶   rÓ  gü©ñÒMbP?g+‡ÙÎ÷ï?r  r7   r  ©rÈ   rÇ   rÉ   )r×   rÈ   rÈ   )r×   rÉ   rÌ   Tr  gèº•Ê€æ?r2   Fg.Lð—`’@rÌ   rÇ   r×   rË   ©ÚhamÚspamr  r  çCTáÏðæç?rÈ   r”   zly_true contains only one label \(2\). Please provide the true labels explicitly through the labels argument.rÏ   zDFound input variables with inconsistent numbers of samples: \[3, 2\]rÁ   rÜ   )rÇ   rÉ   rÈ   ©rÉ   rÇ   rÈ   r  )r<   rŠ   r   r¹   r   Zlogpmfr)   r§   rº   r»   Úlog)	rL   rK   ÚlossZ	loss_truerÓ   Z	error_strZtrue_log_lossZcalculated_log_lossZy_score2rM   rM   rN   Útest_log_loss<
  sR    &ÿ
*


*

ÿ**
r   c                 C   s@   t jddg| d�}t jddg| d�}t||ƒ}t  |¡s<J ‚dS )z¬Check the behaviour internal eps that changes depending on the input dtype.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/24315
    r   r7   r…  N)r<   rŠ   r   Úisfinite)r  rL   rK   r  rM   rM   rN   Útest_log_loss_eps}
  s    
r"  c                 C   sp   t  g d¢¡}t jddgddgddgddgg| d�}tjtd	d
�� t||ƒ W d  ƒ n1 sb0    Y  dS )zGCheck that log_loss raises a warning when y_pred values don't sum to 1.r{  rÈ   rÇ   r×   rÌ   rË   r”   r…  z$The y_pred values do not sum to one.rÏ   N)r<   rŠ   r§   rÞ   rß   r   )r  rL   rK   rM   rM   rN   Ú'test_log_loss_not_probabilities_warning‹
  s    &r#  r
  r³   rˆ   c                 C   s   t | |ƒt d¡ksJ ‚dS )z6Check that log_loss returns 0 for perfect predictions.r   N)r   r§   r¨   r'  rM   rM   rN   Ú!test_log_loss_perfect_predictions•
  s    r$  c            
      C   s¨   t  g d¢¡} t  ddgddgddgddgg¡}ttfg}z"ddlm}m} | ||f¡ W n tyn   Y n0 |D ].\}}|| ƒ||ƒ }}t||ƒ}	t	|	dƒ qtd S )	Nr  rÌ   rÇ   r×   rË   r   )Ú	DataFramer”  r  )
r<   rŠ   r(   r“  r%  r”  r>  ÚImportErrorr   r)   )
Zy_trZy_prÚtypesr%  r”  ZTrueInputTypeZPredInputTyperL   rK   r  rM   rM   rN   Útest_log_loss_pandas_input£
  s    "

r(  c                  C   sú  t  g d¢¡} t  g d¢¡}t | | ¡d t| ƒ }tt| | ƒdƒ tt| |ƒ|ƒ ttd|  |ƒ|ƒ ttd|  d |ƒ|ƒ t t	¡�" t| |dd … ƒ W d   ƒ n1 s´0    Y  t t	¡� t| |d ƒ W d   ƒ n1 sì0    Y  t t	¡� t| |d ƒ W d   ƒ n1 �s&0    Y  t  g d¢¡} t  g d¢¡}d	}tjt	|d
�� t| |ƒ W d   ƒ n1 �s€0    Y  ttdgdgƒdƒ ttdgdgƒdƒ ttdgdgƒdƒ ttdgdgdd�dƒ ttdgdgdd�dƒ d S )N©r   r7   r7   r   r7   r7   ©rÉ   r”   rX   rÌ   r£   gffffffî?r2   ro   r£   r7   )r   r7   r2   r   )r”   r×   rË   rÈ   zMOnly binary classification is supported. The type of the target is multiclassrÏ   r¥   rË   g{®GázÄ?r   r÷  ZfooÚbarrÑ   )
r<   rŠ   r   Znormrd   r*   r   r§   rº   r»   )rL   rK   Z
true_scorer  rM   rM   rN   Útest_brier_score_lossµ
  s0    0,.ÿ*r,  c                  C   sH   d} t jt| d��" tg d¢g d¢ƒ W d   ƒ n1 s:0    Y  d S )Nz%y_pred contains classes not in y_truerÏ   rÃ  rˆ   )r§   rÞ   rß   r   rà   rM   rM   rN   Ú#test_balanced_accuracy_score_unseenØ
  s    r-  zy_true,y_pred)rv   rw   rv   rw   )rv   rv   rv   rw   )rv   rw   rx   rw   c                 C   s˜   t | |dt | ¡d�}tƒ � t| |ƒ}W d   ƒ n1 s<0    Y  |t |¡ksXJ ‚t| |dd�}t| t | | d ¡ƒ}||| d|  ks”J ‚d S )Nr®   rµ   T)Úadjustedr   r7   )r    r<   Úuniquer-   r   r§   r¨   Z	full_like)rL   rK   Zmacro_recallZbalancedr.  ZchancerM   rM   rN   Útest_balanced_accuracy_scoreÞ
  s    	ÿ(r0  r‰   r¬   ))FTrH  )ro   r£   )ÚzeroÚonec                 C   sn   t j d¡}d|d  }}|j||dd�}| tu r@|j|d�}n| ¡ }| |||d�}t  t  |¡¡rjJ ‚dS )	zÀCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    é*   r/  r¥   T)rG  ÚreplacerF  rÑ   N)	r<   r@   rA   Úchoicer   Úuniformrˆ  ÚanyrB  )rm   r¬   rJ   rG   rÒ   rL   rK   rC  rM   rM   rN   Ú*test_classification_metric_pos_label_typesó
  s    r8  zy_true, y_pred, expected_scorero   r£   c                 C   s    t | |dd�t |¡ksJ ‚dS )z•Check the behaviour of `zero_division` for f1-score.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26965
    r£   rA  N)r   r§   r¨   )rL   rK   rÇ  rM   rM   rN   Ú2test_f1_for_small_binary_inputs_with_zero_division  s    r9  ÚscoringrA  r2   )r—   rs   c                 C   s<   t jdd�\}}tddd� ||¡}t|||| ddd� dS )	aZ  Check that we validate `np.nan` properly for classification metrics.

    With `n_jobs=2` in cross-validation, the `np.nan` used for the singleton will be
    different in the sub-process and we should not use the `is` operator but
    `math.isnan`.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/27563
    r   )r6   rª   )Ú	max_depthr6   r2   r~  )r:  Zn_jobsZerror_scoreN)r	   Zmake_classificationr'   rB   r$   )r:  rE   rF   Ú
classifierrM   rM   rN   Ú9test_classification_metric_division_by_zero_nan_validaton&  s    r=  c                  C   sž   t  g d¢¡} t  g d¢¡}d}tjt|d�� t| |d� W d  ƒ n1 sP0    Y  d}tjt|d�� t| ||d� W d  ƒ n1 s�0    Y  dS )	z)Check the message for future deprecation.r)  r*  z$y_prob was deprecated in version 1.5rÏ   )Úy_probNz/`y_prob` and `y_proba` cannot be both specified)r>  Zy_proba)r<   rŠ   r§   rÞ   ÚFutureWarningr   rº   r»   )rL   rK   r+  Ú	error_msgrM   rM   rN   Ú)test_brier_score_loss_deprecation_warning?  s    þ$ýrA  c               	   C   sú  g d¢} g d¢}t  ddgddgddgddgdd	gd
dgg¡}t  ddgddgddgddgddgddgg¡}t| |d�}t| |dd�}t| |dd�}d||  }|t |¡ks²J ‚t  g d¢¡}|d d…  ¡ | ¡  |d d …df< |dd …  ¡ | ¡  |d d …df< t| ||d�}t| ||dd�}t| ||dd�}d||  }|t |¡k�sVJ ‚t  ddgddgddgddgddgddgg¡}t| |ƒ}d|  k �r¦dk �s¬n J ‚t||ƒ}	|	t |¡k�sÊJ ‚t  ddgddgddgddgddgddgg¡}t| |ƒ}|dk �sJ ‚t||ƒ}	|	t |¡k�s.J ‚g d¢} t  ddgddgddgddgddgddgg¡}t| |ƒ}|dk�s|J ‚t||ƒ}	|	dk�s”J ‚g d¢} g d¢}t  ddgddgddgddgg¡}t| |ƒ}|dk�sÞJ ‚t||ƒ}	|	dk�söJ ‚g d¢}t| ||d�}
|
dk�sJ ‚g d¢} g d¢}t  g d ¢g d ¢g d!¢g d"¢g¡}t| |ƒ}d|  k �rndk �stn J ‚t| ||d�}d|  k �ršdk �s n J ‚t  g d#¢g d$¢g d"¢g d%¢g¡}t| |ƒ}|dk �sÚJ ‚t| ||d�}|dk �söJ ‚d S )&Nr(  r  r‰   rX   rÉ   rË   r×   gffffffÖ?gÍÌÌÌÌÌä?r  r  r'  F)rL   rK   r  r7   )r2   r7   rª   r™   rª   r7   rª   r   )rL   rK   r  )rL   rK   r  r  r”   rÈ   r£   r¶   rÓ  )r   r7   r7   r7   )r  r  r  r  )r2   r2   r2   r2   r  )ÚhighrB  ÚlowZneutral)gffffffö?r×   r”   rÈ   )r”   rÉ   rÉ   rÍ   )rÉ   rÉ   r”   )rÈ   r‰   rÌ   r  r  )r<   rŠ   r#   r   r§   r¨   r"  )rL   Zy_true_stringrK   Zy_pred_nullZd2_scoreZlog_likelihoodZlog_likelihood_nullZd2_score_truer  Zd2_score_stringZd2_score_with_sample_weightrM   rM   rN   Útest_d2_log_loss_scoreU  sò    úÿ
úÿ
$$ÿüüúÿ


úÿ


úÿ


"

ÿüÿ
üÿ
rD  c                  C   s*  g d¢} ddgddgddgg}d}t jt|d�� t| |ƒ W d	  ƒ n1 sP0    Y  g d
¢} ddgddgddgg}g d¢}d}t jt|d�� t| ||d� W d	  ƒ n1 s¶0    Y  g d¢} g d¢g d¢g}d}t jt|d�� t| |ƒ W d	  ƒ n1 �s0    Y  dg} ddgg}d}t jt|d�� t| |ƒ W d	  ƒ n1 �sZ0    Y  g d¢} ddgddgddgg}d}t jt|d�� t| |ƒ W d	  ƒ n1 �s¶0    Y  g d¢} dg}ddgddgddgg}d}t jt|d�� t| ||d� W d	  ƒ n1 �s0    Y  d	S )zPTest that d2_log_loss_score raises the appropriate errors on
    invalid inputs.rƒ   rÈ   r”   r‰   rË   r×   z#contain different number of classesrÏ   Nru   z(number of classes in labels is differentrÁ   )r‰   r‰   r‰   )r×   rÌ   rÉ   r  r7   zscore is not well-definedr´   r˜   zy_true contains only one labelz.The labels array needs to contain at least two)r§   rº   r»   r#   rÞ   r   )rL   rK   Úerrra   rM   rM   rN   Útest_d2_log_loss_score_raisesë  s@    (,*
**rF  )NF)°r	  r|   Ú	functoolsr   Ú	itertoolsr   r   r   Únumpyr<   r§   Zscipyr   Zscipy.spatial.distancer   r²  Zscipy.statsr   Zsklearnr	   r
   Zsklearn.datasetsr   Zsklearn.exceptionsr   Zsklearn.metricsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   Zsklearn.metrics._classificationr"   r#   Zsklearn.model_selectionr$   Zsklearn.preprocessingr%   r&   Zsklearn.treer'   Zsklearn.utils._mockingr(   Zsklearn.utils._testingr)   r*   r+   r,   r-   Zsklearn.utils.extmathr.   Zsklearn.utils.fixesr/   r0   Zsklearn.utils.validationr1   rO   rn   rr   ÚmarkZparametrizerp   r‚   r…   r�   r¢   Úfilterwarningsr©   rÀ   rÃ   rÕ   rŠ   rÚ   rÛ   rá   rø   rú   r  r  r  r   r$  r&  r,  r.  r3  r@  rD  rE  rJ  rZ  ra  ri  rq  rx  r|  r}  r  r�  r‚  r‡  r‘  r—  r›  rœ  r�  rŸ  r¥  r§  r¨  r¬  r­  r®  r¯  r´  rº  r¼  rÁ  rÂ  rÆ  rÈ  rÏ  rÔ  r×  rÛ  rÝ  rÞ  rß  rç  rè  rê  rë  rì  rí  rñ  rò  rõ  r  r  r  r  r  r  r   r�  rŒ  Zfloat16r"  r#  r$  r(  r,  r-  r0  r8  r9  r=  rA  rD  rF  rM   rM   rM   rN   Ú<module>   sT  \0A
 ÿ
"



-

üÿþþôþ
ûÿþþóþ
#(þ	
	þû
þû
þû
þû
þûÞþ-
þúÿþ

üþ	
üþ		
'#8
!:

þú
ÿ

#


(M%


B

@
þ^%

#h
=
'
'
&b
 A
	"ýþ#ýþùþÿüþ		üþ	 