a
    �ù»j…O ã                   @   s  d dl Z d dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZmZ d dlmZm Z  d d	l!m"Z" d d
l#m$Z$ d dl%m&Z& d dl'm(Z(m)Z)m*Z*m+Z+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2m3Z3m4Z4 eeegZ5�d?dd„Z6dd„ Z7dd„ Z8dd„ Z9dd„ Z:ej; <dddg¡dd„ ƒZ=dd „ Z>d!d"„ Z?d#d$„ Z@d%d&„ ZAd'd(„ ZBd)d*„ ZCd+d,„ ZDd-d.„ ZEd/d0„ ZFd1d2„ ZGd3d4„ ZHej; <d5e Ig d6¢¡g d7¢fe Ig d6¢¡dfg d8¢g d9¢fg d8¢dfg¡d:d;„ ƒZJej; <d5e Ig d<¢¡g d7¢fe Ig d=¢¡g d>¢fg¡d?d@„ ƒZKej; <d5e Ig dA¢¡dfg dB¢dfg dA¢g d7¢fg dB¢g d9¢fg¡dCdD„ ƒZLej; <dEg dF¢¡dGdH„ ƒZMdIdJ„ ZNej; <dKdLe Ig dA¢¡g dM¢fdLe Ig dB¢¡g dN¢fdOe Ig d<¢¡dfdPe Ig dB¢¡g dQ¢fdRe Ig dA¢¡d dSgfdRe Ig dB¢¡dTdUgfdVe Ig dA¢¡g dW¢fdVe Ig dB¢¡g dX¢fdYe Ig dZ¢¡g d9¢fdYe Ig dX¢¡g d9¢fdYe Ig dW¢¡g d7¢fg¡ej; <d[d\d]g¡d^d_„ ƒƒZOej; <d`dadbd\dcœfdddbd]dcœfded\g dfœfdgd\dhdiœfdji fg¡dkdl„ ƒZPdmdn„ ZQej; <doe5¡dpdq„ ƒZRej; <doe5¡drds„ ƒZSej; Tdt¡ej; <doeeg¡ej; <dudvdwg¡dxdy„ ƒƒƒZUej; <doe5¡dzd{„ ƒZVej; <dddg¡d|d}„ ƒZWd~d„ ZXej; <dddg¡d€d�„ ƒZYd‚dƒ„ ZZd„d…„ Z[d†d‡„ Z\dˆd‰„ Z]dŠd‹„ Z^dŒd�„ Z_ej; <dŽg d�¢g d�¢d gd gfg d�¢g d‘¢d gd gfg d�¢g d’¢d gd gfg d�¢g d“¢d gd gfg d”¢g d�¢dhgd gfg d”¢g d‘¢dhgd gfg d”¢g d’¢dhgd gfg d”¢g d“¢dhgd gfg d•¢g d�¢d–gd gfg d•¢g d‘¢d gd gfg d•¢g d’¢d gd gfg d•¢g d“¢d gd gfg d—¢g d�¢g d˜¢g d•¢fg d—¢g d‘¢g d˜¢g d•¢fg d—¢g d’¢g d˜¢g d•¢fg d—¢g d“¢g d˜¢g d•¢fg d™¢g d�¢g dš¢g d›¢fg d™¢g d‘¢g dš¢g d›¢fg d™¢g d’¢g dš¢g d›¢fg d™¢g d“¢g dš¢g d›¢fg¡dœd�„ ƒZ`ej; <dŽdSd gdhdhgdSgd gfd dSgdhdhgdSgd gfg d�¢g dž¢dhgd gfg d”¢g dž¢dhgd gfg d•¢g dž¢d gd gfg d—¢g dž¢dSgd gfg d™¢g dž¢dSgd gfg dš¢g dž¢dSgd gfg¡dŸd „ ƒZad¡d¢„ Zbej; <d£g d¤¢¡d¥d¦„ ƒZcej; <d§g d¨¢g d©¢g dª¢g d«¢g d¬¢g¡d­d®„ ƒZdej; <d¯d dSgg d�¢d°fg d•¢d dhgd°fg d±¢g d�¢d²fg d³¢g d�¢d²fg d´¢g dµ¢d¶fg¡d·d¸„ ƒZed¹dº„ Zfd»d¼„ Zgd½d¾„ Zhd¿dÀ„ ZidÁdÂ„ ZjdÃdÄ„ ZkdÅdÆ„ Zl�d@dÉdÊ„Zmej; <dËegekejehf¡ej; <dÌeelf¡dÍdÎ„ ƒƒZndÏdÐ„ Zoej; <dÑdÒ¡ej; <dÓdÔ¡ej; <dÕepdSƒ¡dÖd×„ ƒƒƒZqdØdÙ„ ZrdÚdÛ„ ZsdÜdÝ„ Ztej; <dÞg d™¢g dž¢fg d™¢g dž¢gfg d™¢gg dž¢fg¡dßdà„ ƒZudádâ„ Zvej; <dãe0¡dädå„ ƒZwdædç„ Zxdèdé„ Zydêdë„ Zzdìdí„ Z{dîdï„ Z|dðdñ„ Z}dòdó„ Z~dôdõ„ Zej; <döddg¡d÷dø„ ƒZ€dùdú„ Z�dûdü„ Z‚dýdþ„ Zƒdÿ�d „ Z„ej; <�dg dW¢dS�dfg dW¢�ddhfg dW¢�d�dfg¡�d�d„ ƒZ…ej; <�de Ig �d	¢¡dSdSfe Ig �d
¢¡dSdhfe Ig �d
¢¡�ddSfe Ig �d¢¡dSdSfe Ig �d¢¡dSdhfe Ig �d¢¡�ddSfg¡�d�d„ ƒZ†ej; <�de Ig �d¢¡�dg dW¢fe Ig �d¢¡dhg dW¢fe Ig �d¢¡dhg dW¢fe Ig �d¢¡�dg �d¢fg¡ej; <�dddg¡�d�d„ ƒƒZ‡�d�d„ Zˆej; <�dg dW¢dS�dfg dW¢�ddhfg dW¢�ddSfg¡�d�d„ ƒZ‰ej; <�dg dW¢�dfg dW¢dÇfg¡�d�d„ ƒZŠej; <�d g �d!¢g �d"¢g �d#¢g �d$¢g �d%¢gd�d&fg dW¢g �d"¢g �d#¢g �d$¢g �d%¢gd�d'fg �d(¢g �d"¢g �d#¢g �d$¢g �d%¢gg dB¢�d)fg �d(¢g �d"¢g �d#¢g �d$¢g �d%¢gg dQ¢�d*fg �d+¢g �d"¢g �d#¢g �d$¢g �d%¢gg dW¢�d,fg �d+¢g �d"¢g �d#¢g �d$¢g �d%¢gg �d-¢�d.fd dSgg �d/¢g �d0¢gd�d1fg¡�d2�d3„ ƒZ‹ej; <dãe0¡�d4�d5„ ƒZŒej; <�d6eeeeg¡ej; <�d7g �d8¢¡�d9�d:„ ƒƒZ��d;�d<„ ZŽ�d=�d>„ Z�dS (A  é    N)Ústats)ÚdatasetsÚsvm)Úmake_multilabel_classification)ÚUndefinedMetricWarning)ÚLogisticRegression)Úaccuracy_scoreÚaucÚaverage_precision_scoreÚcoverage_errorÚ	dcg_scoreÚ	det_curveÚ%label_ranking_average_precision_scoreÚlabel_ranking_lossÚ
ndcg_scoreÚprecision_recall_curveÚroc_auc_scoreÚ	roc_curveÚtop_k_accuracy_score)Ú_dcg_sample_scoresÚ_ndcg_sample_scores)Útrain_test_split)Úlabel_binarize)Ú_sparse_random_matrix)Ú_convert_containerÚassert_allcloseÚassert_almost_equalÚassert_array_almost_equalÚassert_array_equal)Úsoftmax)ÚCSR_CONTAINERS)Úcheck_arrayÚcheck_consistent_lengthÚcheck_random_stateFc                 C   s  | du rt  ¡ } | j}| j}|r:||dk  ||dk   }}|j\}}t |¡}tdƒ}| |¡ || ||  }}t	|d ƒ}tj
 d¡}tj|| |d| ¡f }tjdddd�}	|	 |d|… |d|… ¡ ||d… ¡}
|rò|
dd…d	f }
|	 ||d… ¡}||d… }|||
fS )
z¼Make some classification predictions on a toy dataset using a SVC

    If binary is True restrict to a binary classification problem instead of a
    multiclass classification problem
    Né   é%   r   éÈ   ZlinearT)ZkernelZprobabilityÚrandom_stateé   )r   Z	load_irisÚdataÚtargetÚshapeÚnpÚaranger#   ÚshuffleÚintÚrandomÚRandomStateZc_Zrandnr   ZSVCÚfitÚpredict_probaZpredict)ZdatasetÚbinaryÚXÚyÚ	n_samplesÚ
n_featuresÚpÚrngZhalfÚclfÚy_scoreÚy_predÚy_true© r?   úHE:\soccer\ml_env\Lib\site-packages\sklearn/metrics/tests/test_ranking.pyÚmake_prediction8   s*    


*rA   c                 C   sd   t  | ¡d }|| |k }|| |k }| dd¡| dd¡ }t  |dk¡}|tt|ƒt|ƒ ƒ S )zKAlternative implementation to check for correctness of
    `roc_auc_score`.r(   éÿÿÿÿr   )r,   ÚuniqueÚreshapeÚsumÚfloatÚlen)r>   r<   Ú	pos_labelÚposÚnegZdiff_matrixZ	n_correctr?   r?   r@   Ú_auch   s    rK   c           	      C   sª   t  | ¡d }t  | |k¡}t  |¡ddd… }|| }| | } d}tt|ƒƒD ]P}| | |krPd}td|d ƒD ]}| | |krr|d7 }qr||d  }||7 }qP|| S )a>  Alternative implementation to check for correctness of
    `average_precision_score`.

    Note that this implementation fails on some edge cases.
    For example, for constant predictions e.g. [0.5, 0.5, 0.5],
    y_true = [1, 0, 0] returns an average precision of 0.33...
    but y_true = [0, 0, 1] returns 1.0.
    r(   NrB   r   ç      ð?)r,   rC   rE   ZargsortÚrangerG   )	r>   r<   rH   Zn_posÚorderÚscoreÚiÚprecÚjr?   r?   r@   Ú_average_precisionw   s    	

rS   c                 C   sd   t | |ƒ\}}}tt|ƒƒ}tt|ƒƒ}d}tdt|ƒƒD ]$}||| || ||d    7 }q:|S )ao  A second alternative implementation of average precision that closely
    follows the Wikipedia article's definition (see References). This should
    give identical results as `average_precision_score` for all inputs.

    References
    ----------
    .. [1] `Wikipedia entry for the Average precision
       <https://en.wikipedia.org/wiki/Average_precision>`_
    r   r(   )r   ÚlistÚreversedrM   rG   )r>   r<   Ú	precisionÚrecallÚ	thresholdZaverage_precisionrP   r?   r?   r@   Ú_average_precision_slow•   s    
"rY   c                 C   s^   dd„ }|| ||ƒ\}}t ||ƒ}d}|}d||  ||  }	|| }
dd||	 |
|	    S )zcAlternative implementation to check for correctness of `roc_auc_score`
    with `max_fpr` set.
    c                 S   sˆ   t | |ƒ\}}}|||k }t ||¡}|||k }t ||k¡}|d }	||	 || g}
||	 || g}t |t ||
|¡¡}||fS )Nr(   )r   r,   ÚappendÚargmaxÚinterp)r>   Ú	y_predictÚmax_fprÚfprÚtprÚ_Únew_fprÚnew_tprZidx_outZidx_inZx_interpZy_interpr?   r?   r@   Ú_partial_roc­   s    z,_partial_roc_auc_score.<locals>._partial_rocr   ç      à?r(   )r	   )r>   r]   r^   rd   rb   rc   Zpartial_aucZfpr1Zfpr2Zmin_areaZmax_arear?   r?   r@   Ú_partial_roc_auc_score¨   s    
rf   ÚdropTc           	      C   sz   t dd�\}}}t||ƒ}t||| d�\}}}t||ƒ}t||dd� t|t||ƒƒ |j|jksfJ ‚|j|jksvJ ‚d S )NT©r4   ©Zdrop_intermediater$   ©Údecimal)rA   rK   r   r	   r   r   r   r+   )	rg   r>   ra   r<   Zexpected_aucr_   r`   Ú
thresholdsÚroc_aucr?   r?   r@   Útest_roc_curveÄ   s    

rn   c                  C   sŒ   t j d¡} t  dgd dgd  ¡}| jddd�}t||dd�\}}}|d dksXJ ‚|d	 dkshJ ‚|j|jksxJ ‚|j|jksˆJ ‚d S )
Nr   é2   r(   é   éd   ©ÚsizeTri   rB   )r,   r0   r1   ÚarrayÚrandintr   r+   )r:   r>   r=   r_   r`   Zthrr?   r?   r@   Útest_roc_curve_end_pointsÒ   s    rv   c            
      C   sŽ   t dd�\} }}t| |ƒ\}}}g }|D ]2}t ||k| @ ¡}t | ¡}	| d| |	 ¡ q(t||dd� |j|jkszJ ‚|j|jksŠJ ‚d S )NTrh   rL   r$   rj   )rA   r   r,   rE   rZ   r   r+   )
r>   ra   r<   r_   r`   rl   Ztpr_correctÚtÚtpr9   r?   r?   r@   Útest_roc_returns_consistencyß   s    
ry   c                  C   sH   t dd�\} }}t t¡� t| |ƒ W d   ƒ n1 s:0    Y  d S )NFrh   )rA   ÚpytestÚraisesÚ
ValueErrorr   )r>   ra   r<   r?   r?   r@   Útest_roc_curve_multiò   s    r}   c                  C   s`   t dd�\} }}t| |d ƒ\}}}t||ƒ}t|ddd� |j|jksLJ ‚|j|jks\J ‚d S )NTrh   re   çÍÌÌÌÌÌì?r$   rj   )rA   r   r	   r   r+   )r>   ra   r<   r_   r`   rl   rm   r?   r?   r@   Útest_roc_curve_confidenceú   s    
r   c                  C   s  t dd�\} }}t | j¡}t| |ƒ\}}}t||ƒ}t|ddd� |j|jksTJ ‚|j|jksdJ ‚t | j¡}t| |ƒ\}}}t||ƒ}t|ddd� |j|jks¨J ‚|j|jks¸J ‚t| |ƒ\}}}t||ƒ}t|ddd� |j|jksðJ ‚|j|jk�sJ ‚d S )NTrh   re   r$   rj   gö(\�Âõè?)rA   r,   Úonesr+   r   r	   r   Úzeros)r>   Úpredr<   Ztrivial_predr_   r`   rl   rm   r?   r?   r@   Útest_roc_curve_hard  s$    


rƒ   c                  C   s  g d¢} g d¢}d}t jt|d��  t| |ƒ\}}}W d   ƒ n1 sH0    Y  t|t t|ƒtj¡ƒ |j	|j	kszJ ‚|j	|j	ksŠJ ‚d}t jt|d��* tdd„ | D ƒ|ƒ\}}}W d   ƒ n1 sÌ0    Y  t|t t|ƒtj¡ƒ |j	|j	ksþJ ‚|j	|j	k�sJ ‚d S )N)
r(   r(   r(   r(   r(   r(   r(   r(   r(   r(   )
r   r(   r   r(   r   r(   r   r(   r   r(   úINo negative samples in y_true, false positive value should be meaningless©ÚmatchúHNo positive samples in y_true, true positive value should be meaninglessc                 S   s   g | ]}d | ‘qS )r(   r?   )Ú.0Úxr?   r?   r@   Ú
<listcomp>5  ó    z,test_roc_curve_one_label.<locals>.<listcomp>)
rz   Úwarnsr   r   r   r,   ÚfullrG   Únanr+   )r>   r=   Úexpected_messager_   r`   rl   r?   r?   r@   Útest_roc_curve_one_label!  s     ÿ.ÿ8r�   c                  C   s¸  ddg} ddg}t | |ƒ\}}}t| |ƒ}t|g d¢ƒ t|g d¢ƒ t|dƒ ddg} ddg}t | |ƒ\}}}t| |ƒ}t|g d¢ƒ t|g d¢ƒ t|dƒ ddg} ddg}t | |ƒ\}}}t| |ƒ}t|ddgƒ t|ddgƒ t|dƒ ddg} ddg}t | |ƒ\}}}t| |ƒ}t|g d¢ƒ t|g d¢ƒ t|dƒ ddg} ddg}t | |ƒ\}}}t| |ƒ}t|ddgƒ t|ddgƒ t|dƒ ddg} dd	g}d
}tjt|d��  t | |ƒ\}}}W d   ƒ n1 �sÚ0    Y  t|g d¢ƒ t|tjtjtjgƒ d}tjt|d�� t| |ƒ}W d   ƒ n1 �s<0    Y  t	 
|¡�sVJ ‚ddg} dd	g}d}tjt|d��  t | |ƒ\}}}W d   ƒ n1 �s 0    Y  t|tjtjtjgƒ t|g d¢ƒ d}tjt|d�� t| |ƒ}W d   ƒ n1 �s0    Y  t	 
|¡�sJ ‚t ddgddgg¡} t ddgddgg¡}tjt|d�� t| |dd� W d   ƒ n1 �s|0    Y  tjt|d�� t| |dd� W d   ƒ n1 �sº0    Y  tt| |dd�dƒ tt| |dd�dƒ t ddgddgg¡} t ddgddgg¡}tjt|d�� t| |dd� W d   ƒ n1 �sL0    Y  tjt|d�� t| |dd� W d   ƒ n1 �sŠ0    Y  tt| |dd�dƒ tt| |dd�dƒ t ddgddgg¡} t ddgddgg¡}tt| |dd�dƒ tt| |dd�dƒ tt| |dd�dƒ tt| |dd�dƒ t ddgddgg¡} t ddgddgg¡}tt| |dd�dƒ tt| |dd�dƒ tt| |dd�dƒ tt| |dd�dƒ d S )Nr   r(   ©r   r   r(   ©r   r(   r(   rL   ç        re   ç      Ð?ç      è?r‡   r…   ©r“   re   rL   úOOnly one class is present in y_true. ROC AUC score is not defined in that case.r„   Úmacro©ÚaverageÚweightedÚsamplesÚmicro)r   r   r   r   rz   rŒ   r   r,   rŽ   ÚmathÚisnanrt   )r>   r<   r`   r_   ra   rm   r�   r	   r?   r?   r@   Útest_roc_curve_toydata<  s²    









ÿ0ÿ*ÿ0ÿ*....r    c                  C   sx   g d¢} g d¢}t | |dd�\}}}t|tjdddgƒ g d¢} g d	¢}t | |dd�\}}}t|tjdd
dddgƒ d S )N©r   r   r   r   r(   r(   ©r“   çš™™™™™É?re   ç333333ã?çffffffæ?rL   Tri   rL   r¥   r“   ©r   r   r   r   r   r   r   r(   r(   r(   r(   r(   r(   ©r“   çš™™™™™¹?r¤   r¤   r¥   çš™™™™™é?r~   r¤   r¥   r©   r~   r~   rL   r~   r¤   )r   r   r,   Úinf)r>   r<   r`   r_   rl   r?   r?   r@   Ú test_roc_curve_drop_intermediate®  s    r«   c                  C   sh   g d¢} g d¢}t  dd¡}t| ||d�\}}}t  |¡dk  ¡ dksJJ ‚t  |¡dk  ¡ dksdJ ‚d S )N©r   r   r(   r(   r(   )r¨   r¥   ç333333Ó?çš™™™™™Ù?re   r£   é   ©Úsample_weightr   )r,   Úrepeatr   ÚdiffrE   )r>   r<   r±   r_   r`   ra   r?   r?   r@   Ú!test_roc_curve_fpr_tpr_increasing¼  s    r´   c                  C   s¤   ddg} ddg}t t| |ƒdƒ ddg} ddg}t t| |ƒdƒ g d¢} g d¢}t t| |ƒdƒ ddg} ddg}t t| |ƒdƒ g d¢} g d¢}t t| |ƒdƒ d S )Nr   r(   re   ©r(   r   r   r’   ©r   re   r(   )r   r	   )r‰   r6   r?   r?   r@   Útest_aucÈ  s    r·   c                  C   sÖ   t  t¡�" tg d¢ddgƒ W d   ƒ n1 s20    Y  t  t¡� tdgdgƒ W d   ƒ n1 sj0    Y  g d¢} g d¢}d t | ¡¡}t jtt |¡d�� t| |ƒ W d   ƒ n1 sÈ0    Y  d S )	Nr–   r¨   r£   r“   )r$   r(   rp   é   )r¯   é   é   é   z+x is neither increasing nor decreasing : {}r…   )	rz   r{   r|   r	   Úformatr,   rt   ÚreÚescape)r‰   r6   Úerror_messager?   r?   r@   Útest_auc_errorsÛ  s    0,rÀ   zy_true, labels)r   r(   r   r$   ©r   r(   r$   )ÚaÚbrÂ   Úc)rÂ   rÃ   rÄ   c                 C   sZ  t  g d¢g d¢g d¢g d¢g¡}tg d¢g d¢ƒ}tg d¢g d¢ƒ}|| d	 }tg d
¢g d¢ƒ}tg d¢g d¢ƒ}|| d	 }tddgddgƒ}	tddgddgƒ}
|	|
 d	 }|| | d }tt| ||dd�|ƒ |||g}g d¢}t j||d�}tt| ||ddd�|ƒ d}tjt|d��" t| ||dd d� W d   ƒ n1 �sL0    Y  d S )N©r¨   r©   r¨   ©r­   r®   r­   ©çffffffÖ?re   ç333333Ã?©r   r£   r©   ©r(   r   r(   )r¨   r­   rÈ   ©r   r(   r   )r©   r®   re   r$   ©r(   r(   r   )r¨   rÈ   r   r‘   )r¨   rÉ   r©   r(   r   r®   r£   r­   r©   rp   Úovo©ÚlabelsÚmulti_class)r•   r•   re   )Úweightsr›   ©rÐ   rÑ   rš   z6average=None is not implemented for multi_class='ovo'.r…   )r,   rt   r   r   rš   rz   r{   ÚNotImplementedError)r>   rÐ   Úy_scoresÚscore_01Úscore_10Zaverage_score_01Zscore_02Zscore_20Zaverage_score_02Zscore_12Zscore_21Zaverage_score_12Zovo_unweighted_scoreZpair_scoresZ
prevalenceZovo_weighted_scorer¿   r?   r?   r@   Ú#test_multiclass_ovo_roc_auc_toydataì  s:    ÿþ

ÿürØ   )r   r$   r   r$   )rÂ   ÚdrÂ   rÙ   )rÂ   rÃ   rÙ   c                 C   s„   t  g d¢g d¢g d¢g d¢g¡}tg d¢g d¢ƒ}tg d¢g d¢ƒ}|| d	 }tt| ||d
d�|ƒ tt| ||d
dd�|ƒ d S )N)r£   r“   r©   )r¤   r“   r®   )çš™™™™™á?r“   çÍÌÌÌÌÌÜ?)r®   r“   r¤   )r(   r   r(   r   )r£   r¤   rÚ   r®   )r   r(   r   r(   )r©   r®   rÛ   r¤   r$   rÎ   rÏ   r›   rÓ   ©r,   rt   r   r   )r>   rÐ   rÕ   rÖ   r×   Z	ovo_scorer?   r?   r@   Ú*test_multiclass_ovo_roc_auc_toydata_binary'  s    ÿÿ
ÿürÝ   )r   r(   r$   r$   )rÂ   rÃ   rÄ   rÄ   c                 C   sè   t  g d¢g d¢g d¢g d¢g¡}tg d¢|d d …df ƒ}tg d¢|d d …df ƒ}tg d	¢|d d …d
f ƒ}tt| |d|d d�|||gƒ || | d }tt| |d|d�|ƒ |d |d  |d  }tt| |d|dd�|ƒ d S )N©rL   r“   r“   )r¨   re   r®   )r¨   r¨   r©   )r­   r­   r®   ©r(   r   r   r   r   )r   r(   r   r   r(   ©r   r   r(   r(   r$   Úovr)rÑ   rÐ   rš   g      @)rÑ   rÐ   r”   re   r›   rÜ   )r>   rÐ   rÕ   Zout_0Zout_1Zout_2Zresult_unweightedZresult_weightedr?   r?   r@   Ú#test_multiclass_ovr_roc_auc_toydataK  s,    ÿþþ
ÿürâ   zmulti_class, average))rá   r˜   )rá   r�   )rÎ   r˜   c                 C   s¤   t  g d¢¡}g d¢g d¢g d¢g d¢g}tt||| |d�dƒ g d¢g d¢g d¢g d¢g}t||| |d�dk stJ ‚dt  d	¡ }t||| |d�t d
¡ks J ‚d S )N)rp   r(   r$   r   )r“   r“   r“   rL   )r“   rL   r“   r“   )r“   r“   rL   r“   )r•   çš™™™™™©?rã   rÉ   ©rÑ   rš   rL   r”   )r¸   r¸   re   )r,   rt   r   r   r€   rz   Úapprox)rÑ   rš   r>   Z	y_perfectZy_imperfectZy_chancer?   r?   r@   Ú0test_perfect_imperfect_chance_multiclass_roc_aucv  s2    	üþüÿÿÿþræ   c           	         sŠ   | ‰ t jjg d¢dˆ d�}t ‡ fdd„|D ƒ¡}t|g d¢d�}t| ¡ | ¡ ƒ\}}}t||ƒ}t	||dd	d
�}|t
 |¡ks†J ‚d S )N)ç       @rL   re   éè  )rs   r'   c                    s"   g | ]}t jjd |ˆ d� ¡ ‘qS )r(   )Únr9   r'   )r   ZmultinomialÚrvsr[   )rˆ   Zy_pred_i©Úseedr?   r@   rŠ   ¨  s   ÿz3test_micro_averaged_ovr_roc_auc.<locals>.<listcomp>rÁ   )Úclassesrá   r�   rä   )r   Z	dirichletrê   r,   Úasarrayr   r   Zravelr	   r   rz   rå   )	Úglobal_random_seedr=   r>   Zy_onehotr_   r`   ra   Zroc_auc_by_handZroc_auc_autor?   rë   r@   Útest_micro_averaged_ovr_roc_auc   s    
þÿ
rð   zmsg, y_true, labelsz!Parameter 'labels' must be unique©r   r$   r   )rÂ   rÂ   rÃ   zKNumber of classes in y_true not equal to the number of columns in 'y_score'z"Parameter 'labels' must be ordered)rÂ   rÄ   rÃ   zMNumber of given labels, 2, not equal to the number of columns in 'y_score', 3r(   rÂ   rÃ   zMNumber of given labels, 4, not equal to the number of columns in 'y_score', 3)r   r(   r$   rp   )rÂ   rÃ   rÄ   rÙ   z2'y_true' contains labels not in parameter 'labels')rÂ   rÃ   rÄ   ÚerÑ   rÎ   rá   c                 C   sd   t  g d¢g d¢g d¢g d¢g¡}tjt| d��  t||||d� W d   ƒ n1 sV0    Y  d S )NrÅ   rÆ   rÇ   rÊ   r…   rÏ   )r,   rt   rz   r{   r|   r   )Úmsgr>   rÐ   rÑ   rÕ   r?   r?   r@   Ú*test_roc_auc_score_multiclass_labels_error´  s
    Iÿrô   zmsg, kwargszLaverage must be one of \('macro', 'weighted', None\) for multiclass problemsrœ   )rš   rÑ   zUaverage must be one of \('micro', 'macro', 'weighted', None\) for multiclass problemszksample_weight is not supported for multiclass one-vs-one ROC AUC, 'sample_weight' must be None in this case)rÑ   r±   z|Partial AUC computation not available in multiclass setting, 'max_fpr' must be set to `None`, received `max_fpr=0.5` insteadre   )rÑ   r^   z'multi_class must be in \('ovo', 'ovr'\)c                 C   sp   t dƒ}| dd¡}t|ƒ}|jdddd�}tjt| d��" t||fi |¤Ž W d   ƒ n1 sb0    Y  d S )Né”  é   rp   r   rr   r…   )r#   Úrandr   ru   rz   r{   r|   r   )ró   Úkwargsr:   r<   Zy_probr>   r?   r?   r@   Ú#test_roc_auc_score_multiclass_error  s    'rù   c                  C   sî   t dƒ} |  d¡}tjddd�}d}tjt|d�� t||ƒ W d   ƒ n1 sR0    Y  tjddd�}tjt|d�� t||ƒ W d   ƒ n1 s˜0    Y  tj	dddd�}tjt|d�� t||ƒ W d   ƒ n1 sà0    Y  d S )Nrõ   é
   r/   ©Zdtyper—   r…   rB   )
r#   r÷   r,   r�   rz   rŒ   r   r   r€   r�   )r:   r=   r>   Úwarn_messager?   r?   r@   Útest_auc_score_non_binary_class4  s    
ÿ((rý   Ú
curve_funcc                 C   sb   t dƒ}|jdddd�}| d¡}d}tjt|d�� | ||ƒ W d   ƒ n1 sT0    Y  d S )Nrõ   r   rp   rú   rr   z"multiclass format is not supportedr…   )r#   ru   r÷   rz   r{   r|   )rþ   r:   r>   r=   ró   r?   r?   r@   Ú&test_binary_clf_curve_multiclass_errorI  s    
rÿ   c                 C   sâ   d}t jt|d��, | tjddgdd�ddgƒ W d   ƒ n1 sD0    Y  t jt|d��, | tjddgtd�ddgƒ W d   ƒ n1 sŽ0    Y  g d	¢}| g d
¢|ƒ}| g d¢|ƒ}t||ƒD ]\}}tj ||¡ qÆd S )Nz‘y_true takes value in {'a', 'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r…   rÂ   rÃ   z<U1rû   r“   rL   )r“   rL   r£   gáz®GáÚ?)r   r(   r(   r   )r“   rL   rL   r“   )	rz   r{   r|   r,   rt   ÚobjectÚzipZtestingr   )rþ   ró   r=   Z	int_curveZfloat_curveZint_curve_partZfloat_curve_partr?   r?   r@   Ú(test_binary_clf_curve_implicit_pos_labelS  s    ÿ::r  z.ignore:Support for labels represented as bytesÚlabels_typerT   rt   c                 C   sR   t ddg|ƒ}d}tjt|d�� | |ddgƒ W d   ƒ n1 sD0    Y  d S )Nó   aó   bz“y_true takes value in {b'a', b'b'} and pos_label is not specified: either make y_true take value in {0, 1} or {-1, 1} or pass pos_label explicitly.r…   r“   rL   )r   rz   r{   r|   )rþ   r  rÐ   ró   r?   r?   r@   Ú.test_binary_clf_curve_implicit_bytes_pos_labelm  s
    ÿr  c                 C   sn   g d¢}g d¢}g d¢}| |||d�}| |d d… |d d… |d d… d�}t ||ƒD ]\}}t||ƒ qVd S )Nr¬   )r¨   r£   r­   r®   re   )r(   r(   r(   re   r   r°   rB   )r  r   )rþ   r>   r<   r±   Zresult_1Zresult_2Zarr_1Zarr_2r?   r?   r@   Ú(test_binary_clf_curve_zero_sample_weight}  s    &r  c           
      C   sF  t dd�\}}}t||| ƒ t|dd … |dd … | d�\}}}|d dksPJ ‚|d |dd …  ¡ kslJ ‚d|t |dk¡< | ¡ }t||| ƒ t||ƒ g d¢}g d	¢}	t||	| d�\}}}| rðt|g d
¢ƒ t|g d¢ƒ t|g d¢ƒ n*t|g d¢ƒ t|g d¢ƒ t|g d	¢ƒ |j	|j	k�s,J ‚|j	|j	d k�sBJ ‚d S )NTrh   r(   ri   r   rL   rB   ©r(   r   r   r(   )r(   r$   rp   r¸   )re   çÚÁQUUÕ?rL   rL   )rL   re   re   r“   )r(   r$   r¸   )re   r	  re   rL   rL   )rL   re   re   re   r“   )
rA   Ú_test_precision_recall_curver   Úmeanr,   ÚwhereÚcopyr   r   rs   )
rg   r>   ra   r<   r9   Úrrw   Zy_true_copyrÐ   Zpredict_probasr?   r?   r@   Útest_precision_recall_curveŠ  s*    $
r  c                 C   s´   t | ||d�\}}}t| |ƒ}t|ddƒ t|t| |ƒƒ tt| |ƒ|dd� |j|jks^J ‚|j|jd ksrJ ‚t | t |¡|d�\}}}|j|jksœJ ‚|j|jd ks°J ‚d S )Nri   g°rh‘í|ë?rp   r$   rj   r(   )	r   rY   r   r
   r   rS   rs   r,   Ú
zeros_like)r>   r<   rg   r9   r  rl   Zprecision_recall_aucr?   r?   r@   r
  ª  s     

ÿÿÿr
  c              	   C   s²  t jdd���ð ddg}ddg}t||| d�\}}}t||ƒ}t|g d¢ƒ t|g d¢ƒ t|dƒ ddg}ddg}t||| d�\}}}t||ƒ}t|g d	¢ƒ t|g d
¢ƒ t|dƒ ddg}ddg}t||| d�\}}}t||ƒ}t|ddgƒ t|ddgƒ t|dƒ ddg}ddg}t||| d�\}}}t||ƒ}t|g d¢ƒ t|g d¢ƒ t|dƒ ddg}ddg}t||| d�\}}}t||ƒ}t|ddgƒ t|ddgƒ t|dƒ ddg}ddg}tjtdd��$ t||| d�\}}}W d   ƒ n1 �sþ0    Y  tjtdd�� t||ƒ}W d   ƒ n1 �s80    Y  t	|g d¢ƒ t	|g d¢ƒ t	|dƒ ddg}ddg}t||| d�\}}}tt||ƒdƒ t|g d¢ƒ t|g d¢ƒ t  
ddgddgg¡}t  
ddgddgg¡}tjtdd��$ t	t||dd�dƒ W d   ƒ n1 �s0    Y  tjtdd��$ t	t||dd�dƒ W d   ƒ n1 �sb0    Y  t	t||dd�dƒ t	t||dd�dƒ t  
ddgddgg¡}t  
ddgddgg¡}tjtdd��$ t	t||dd�dƒ W d   ƒ n1 �sú0    Y  tjtdd��$ t	t||dd�dƒ W d   ƒ n1 �s>0    Y  t	t||dd�dƒ t	t||dd�dƒ t  
ddgddgg¡}t  
ddgddgg¡}tt||dd�dƒ tt||dd�dƒ tt||dd�dƒ tt||dd�dƒ t  
ddgddgg¡}t  
ddgddgg¡}tjtdd��$ t	t||dd�dƒ W d   ƒ n1 �sR0    Y  t	t||dd�dƒ tjtdd��$ t	t||dd�dƒ W d   ƒ n1 �sª0    Y  tjtdd��$ t	t||dd�dƒ W d   ƒ n1 �sî0    Y  t  
ddgddgg¡}t  
ddgddgg¡}t	t||dd�dƒ t	t||dd�dƒ t	t||dd�dƒ t	t||dd�dƒ t  
ddgddgg¡}t  
ddgddgg¡}tt||dd�dƒ tt||dd�dƒ tt||dd�dƒ tt||dd�dƒ W d   ƒ n1 �s0    Y  t jdd��€ t  
ddgddgg¡}t  
ddgddgg¡}tjtdd��$ t	t||dd�dƒ W d   ƒ n1 �s„0    Y  W d   ƒ n1 �s¤0    Y  d S )NÚraise)Úallr   r(   ri   )re   r(   r(   rÍ   rL   )re   r“   rL   rÞ   re   r“   r”   r•   z!No positive class found in y_truer…   r‘   )rL   rL   rL   )r(   re   r“   r˜   r™   r›   rœ   r�   Úignore)r,   Zerrstater   r
   r   r   rz   rŒ   ÚUserWarningr   rt   )rg   r>   r<   r9   r  ra   Zauc_prcr?   r?   r@   Ú#test_precision_recall_curve_toydataÀ  s   









4*
ÿ$ÿ$ÿÿ$ÿ$ÿÿÿÿÿÿ$ÿÿ$ÿ$ÿÿÿÿÿÿ$ÿr  c                  C   sÌ   g d¢} g d¢}t | |dd�\}}}t|g d¢ƒ g d¢} g d¢}t | |dd�\}}}t|g d¢ƒ g d	¢} g d
¢}t | |dd�\}}}t|ddgƒ g d¢} g d
¢}t | |dd�\}}}t|g d
¢ƒ dS )z9Check the behaviour of the `drop_intermediate` parameter.r¡   r¢   Tri   )r“   r¥   rL   r¦   r§   )r“   r¤   r¥   r©   r~   rL   ©r   r   r   r   )r“   r¨   r£   r­   r“   r­   ©r(   r(   r(   r(   N)r   r   )r>   r<   rV   rW   rl   r?   r?   r@   Ú-test_precision_recall_curve_drop_intermediated  s0    ÿÿÿÿr  c                  C   s<   t jdtd�} d| d d d…< t  d¡}t| |ƒdks8J ‚d S )Nrq   rû   r(   r¸   r”   )r,   r�   r/   r€   r
   ©r>   r<   r?   r?   r@   Ú&test_average_precision_constant_values†  s    
r  c                  C   s`   t  ddg¡} t  ddg¡}d}tjt|d�� t| |dd� W d   ƒ n1 sR0    Y  d S )Nr   r(   z>pos_label=2 is not a valid label. It should be one of \[0, 1\]r…   r$   ©rH   ©r,   rt   rz   r{   r|   r
   ©r>   r=   Úerr_msgr?   r?   r@   Ú4test_average_precision_score_binary_pos_label_errors”  s
    r  c                  C   sˆ   t  ddgddgddgddgg¡} t  ddgddgddgddgg¡}d}tjt|d�� t| |dd	� W d   ƒ n1 sz0    Y  d S )
Nr(   r   r~   r¨   r©   r£   znParameter pos_label is fixed to 1 for multilabel-indicator y_true. Do not set pos_label or set pos_label to 1.r…   r  r  r  r?   r?   r@   Ú8test_average_precision_score_multilabel_pos_label_errors�  s    ""ÿr   c               	   C   s€   t  g d¢¡} t  g d¢g d¢g d¢g d¢g d¢g d¢g¡}d}tjt|d�� t| |dd	� W d   ƒ n1 sr0    Y  d S )
N)r   r(   r$   r   r(   r$   )re   r£   r¨   )r®   re   r­   )r¨   r£   r¤   )r£   r­   re   zdParameter pos_label is fixed to 1 for multiclass y_true. Do not set pos_label or set pos_label to 1.r…   rp   r  r  r  r?   r?   r@   Ú8test_average_precision_score_multiclass_pos_label_errorsª  s    úÿÿr!  c                  C   sÄ   t dd�\} }}t| |ƒ}t| d| ƒ}t| d| ƒ}t| |d ƒ}||ksPJ ‚||ks\J ‚||kshJ ‚t| |ƒ}t| d| ƒ}t| d| ƒ}	t| |d ƒ}
||ks¨J ‚||	ks´J ‚||
ksÀJ ‚d S )NTrh   rq   g�íµ ÷Æ°>rú   )rA   r   r
   )r>   ra   r<   rm   Zroc_auc_scaled_upZroc_auc_scaled_downZroc_auc_shiftedZpr_aucZpr_auc_scaled_upZpr_auc_scaled_downZpr_auc_shiftedr?   r?   r@   Útest_score_scale_invariance¿  s    

r"  z(y_true,y_score,expected_fpr,expected_fnrr‘   r¶   )r   r”   re   )re   r•   r(   ©r”   re   r•   rÌ   r’   r“   rµ   )r(   r(   re   rË   rÍ   )r   re   re   c                 C   s(   t | |ƒ\}}}t||ƒ t||ƒ d S ©N©r   r   ©r>   r<   Zexpected_fprZexpected_fnrr_   Úfnrra   r?   r?   r@   Útest_det_curve_toydataØ  s    
r(  ©r”   re   re   c                 C   s(   t | |ƒ\}}}t||ƒ t||ƒ d S r$  r%  r&  r?   r?   r@   Útest_det_curve_tie_handlingù  s    
r*  c                   C   s*   t tg d¢g d¢ƒtg d¢g d¢ƒƒ d S )Nr‘   r¶   r¡   )r   r   re   re   r(   r(   )r   r   r?   r?   r?   r@   Útest_det_curve_sanity_check  s    þr+  r<   )r   r”   re   r•   r(   c                 C   sF   t g d¢t d| ¡d�\}}}t|dgƒ t|dgƒ t|| gƒ d S )N)r   r(   r   r(   r   r(   r¹   r  r(   r   )r   r,   r�   r   )r<   r_   r'  rX   r?   r?   r@   Útest_det_curve_constant_scores  s    ÿr,  r>   )r   r   r   r   r   r(   r¡   )r   r   r   r(   r(   r(   )r   r   r(   r(   r(   r(   )r   r(   r(   r(   r(   r(   c                 C   s.   t | | d�\}}}t|dgƒ t|dgƒ d S )Nr  r   r%  )r>   r_   r'  ra   r?   r?   r@   Útest_det_curve_perfect_scores   s    r-  zy_true, y_pred, err_msgzinconsistent numbers of samples©r   r   r   z#Only one class is present in y_true©r(   r(   r(   )Úcancerr0  ú
not cancer)r£   r­   r©   zpos_label is not specifiedc                 C   s<   t jt|d�� t| |ƒ W d   ƒ n1 s.0    Y  d S )Nr…   )rz   r{   r|   r   r  r?   r?   r@   Útest_det_curve_bad_input1  s    r2  c            	      C   sª   dgd dgd  } t  g d¢¡}d| }t| |dd�\}}}t| |dd�\}}}|d t d	¡kshJ ‚|d t d
¡ks~J ‚t||d d d… ƒ t||d d d… ƒ d S )Nr0  rp   r1  rº   )
r¨   r®   r¤   r£   r­   r®   re   r¤   r¥   r~   r(   r  r   r®   r£   rB   )r,   rt   r   rz   rå   r   )	r>   Zy_pred_pos_not_cancerZy_pred_pos_cancerZfpr_pos_cancerZfnr_pos_cancerZth_pos_cancerZfpr_pos_not_cancerZfnr_pos_not_cancerZth_pos_not_cancerr?   r?   r@   Útest_det_curve_pos_labelE  s"    ýýr3  c                 C   sØ  t | ddggddggƒdƒ t | ddggddggƒdƒ t | ddggddggƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d	¢gg d¢gƒdƒ t | g d
¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d	¢gg d¢gƒdƒ t | g d
¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d	¢gg d¢gƒdƒ t | g d
¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | ddggddggƒdƒ t | ddggddggƒdƒ t | ddggddggƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d	¢gg d¢gƒdƒ t | g d
¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ t | g d¢gg d¢gƒdƒ d S )Nr   r(   r”   r•   re   r‘   r#  rÌ   r’   rµ   gUUUUUUÕ?rË   gªªªªªªê?rÍ   gªªªªªªâ?©r•   re   r”   r/  ©re   r•   r”   r)  )re   re   re   çUUUUUUå?)r(   r(   r(   r   )re   re   re   re   )r   ©Ú
lrap_scorer?   r?   r@   Úcheck_lrap_toy_  s^    ÿÿÿÿÿÿr9  c                 C   sÊ   t dƒ}tddƒD ]‚}|jd|fd�}t |¡}t d|f¡}| ||ƒdksPJ ‚| ||ƒdksbJ ‚t d|f¡}| ||ƒdks‚J ‚| ||ƒdksJ ‚qt| dgdgdgdggdgdgdgdggƒdƒ d S )Nr   r$   r¯   r(   rr   rL   re   )r#   rM   Úuniformr,   r  r�   r€   r   )r8  r'   Ún_labelsr<   Zy_score_tiesr>   r?   r?   r@   Ú!check_zero_or_all_relevant_labelsš  s    
*ÿr<  c                 C   sˆ  t  t¡�" | g d¢g d¢ƒ W d   ƒ n1 s20    Y  t  t¡�0 | g d¢g d¢g d¢g d¢gƒ W d   ƒ n1 s|0    Y  t  t¡�0 | g d¢g d¢g d¢g d¢gƒ W d   ƒ n1 sÆ0    Y  t  t¡�* | ddgddggddgƒ W d   ƒ n1 �s0    Y  t  t¡�, | ddgddggddggƒ W d   ƒ n1 �sT0    Y  t  t¡�. | ddgddggdgdggƒ W d   ƒ n1 �sž0    Y  t  t¡�, | ddggddgddggƒ W d   ƒ n1 �sæ0    Y  t  t¡�. | dgdggddgddggƒ W d   ƒ n1 �s00    Y  t  t¡�. | ddgddggdgdggƒ W d   ƒ n1 �sz0    Y  d S )	NrÌ   )r”   r­   r£   rÁ   )r”   r•   r“   )r¥   r­   r“   )r©   r£   r“   r   r(   )rz   r{   r|   r7  r?   r?   r@   Úcheck_lrap_error_raised±  s(    0>ÿ":<><>r=  c              	   C   sz   t ddƒD ]j}t d|f¡}t d|ƒD ]L}t || ƒD ]:}t d|f¡}d|d||| …f< t| ||ƒ|| ƒ q6q&q
d S )Nr$   rú   r(   r   )rM   r,   r€   r�   r   )r8  r;  r<   Ú
n_relevantrI   r>   r?   r?   r@   Úcheck_lrap_only_tiesË  s    r?  c              	      sÖ   t ddƒD ]Æ}|t |¡ d|f¡d  }t d|f¡}d|d< d|d< t| ||ƒd| d d ƒ t d|ƒD ]`‰ t |ˆ  ƒD ]N‰t d|f¡}d|dˆˆˆ  …f< t| ||ƒt‡ ‡fdd„t ˆ ƒD ƒƒƒ q~qnq
d S )	Nr$   rú   r(   )r   r   )r   rB   r   c                 3   s&   | ]}|d  ˆ| d  ˆ   V  qdS )r(   Nr?   ©rˆ   r  ©r>  rI   r?   r@   Ú	<genexpr>î  s   ÿz>check_lrap_without_tie_and_increasing_score.<locals>.<genexpr>)rM   r,   r-   rD   r�   r   rE   )r8  r;  r<   r>   r?   rA  r@   Ú+check_lrap_without_tie_and_increasing_scoreÚ  s     þþrC  c                    s
  t | |ƒ t| ƒ} t|ƒ}| j\}}t |f¡}t|ƒD ]È}tj|| dd�\}}|j}|| ‰tjˆ|d d� 	¡ }	|	ˆ ‰| |  
¡ d }
|
jdks¢|
j|kr¬d||< q8d||< |
D ]4‰ t‡ ‡fdd„|
D ƒƒ}||  |ˆˆ   7  < q¸||  |
j  < q8| ¡ S )	z8Simple implementation of label ranking average precisionT)Zreturn_inverser(   )Z	minlengthr   r“   c                 3   s   | ]}ˆ| ˆˆ  kV  qd S r$  r?   r@  ©ÚlabelZrankr?   r@   rB    r‹   z_my_lrap.<locals>.<genexpr>)r"   r!   r+   r,   ÚemptyrM   rC   rs   ZbincountZcumsumZnonzerorE   r  )r>   r<   r7   r;  rO   rP   Zunique_rankZinv_rankZn_ranksZ	corr_rankZrelevantZn_ranked_abover?   rD  r@   Ú_my_lrapõ  s*    

rG  r¯   rö   c           	      C   sš   t dd|||d�\}}t|jd |jd |d�}t|dƒrB| ¡ }t||ƒ}t||ƒ}t||ƒ t|ƒ}|j	||fd�}t||ƒ}t||ƒ}t||ƒ d S )Nr(   F)r8   Zallow_unlabeledr'   Ú	n_classesr7   r   )Zn_componentsr8   r'   Útoarrayrr   )
r   r   r+   ÚhasattrrI  r   rG  r   r#   r:  )	r8  rH  r7   r'   ra   r>   r<   Z
score_lrapZscore_my_lrapr?   r?   r@   Ú%check_alternative_lrap_implementation  s,    û
	ý





rK  ÚcheckÚfuncc                 C   s   | |ƒ d S r$  r?   )rL  rM  r?   r?   r@   Útest_label_ranking_avp;  s    rN  c                   C   s   t tƒ d S r$  )r=  r   r?   r?   r?   r@   Útest_lrap_error_raisedI  s    rO  r7   )r(   r$   r»   rö   rH  )r$   r¯   rú   r'   c                 C   s   t t|| |ƒ d S r$  )rK  r   )r7   rH  r'   r?   r?   r@   Ú$test_alternative_lrap_implementationM  s    ÿrP  c                  C   s„   t jg d¢g d¢g d¢gtd�} t  g d¢g d¢g d¢g¡}t  g d¢¡}t  g d	¢¡}tt| ||d
�t  || ¡t  |¡ ƒ d S )Nrß   r  r  rû   )r­   r®   r£   r¨   )r¨   r£   r­   r®   ©r®   r­   r£   r¨   )re   r•   rL   )rL   rL   r“   r°   )r,   rt   Úboolr   r   rE   )r>   r<   Zsamplewise_lrapsr±   r?   r?   r@   Ú&test_lrap_sample_weighting_zero_labelsV  s     ÿÿürS  c                   C   s¤  t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d	¢gg d¢gƒdƒ t tg d
¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d	¢gg d¢gƒdƒ t tg d
¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d	¢gg d¢gƒdƒ t tg d
¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d	¢g d¢gg d¢g d¢gƒdƒ t tg d	¢g d¢g d
¢gg d¢g d¢g d¢gƒdƒ t tg d	¢g d¢g d
¢gg d¢g d¢g d¢gƒdƒ d S )Nr   r(   r”   r•   r$   r.  r#  r‘   rÌ   r’   rµ   rp   rË   rÍ   r/  r4  r5  ©r¨   g      $@éýÿÿÿ©r   r(   rp   rç   ©r¨   rú   rU  rñ   g«ªªªªª@©rp   r(   rp   ©r   r   r?   r?   r?   r@   Útest_coverage_errork  sX     þ(ÿü(ÿürZ  c                   C   sT  t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d	¢gg d¢gƒdƒ t tg d
¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ d S )Nr   re   r(   r$   r.  r)  r‘   rÌ   r’   rµ   rp   rË   rÍ   r/  rY  r?   r?   r?   r@   Útest_coverage_tie_handling¢  s    r[  zy_true, y_scorec                 C   s<   t jtdd�� t| |ƒ W d   ƒ n1 s.0    Y  d S )Nz'Expected 2D array, got 1D array insteadr…   )rz   r{   r|   r   r  r?   r?   r@   Útest_coverage_1d_error_message²  s    r\  c                   C   sT  t tddggddggƒdƒ t tddggddggƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d	¢gg d¢gƒdƒ t tg d
¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tddggddggƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢g d¢gg d¢g d¢gƒdƒ t tg d¢g d¢g d	¢gg d¢g d¢g d¢gƒdƒ t tg d¢g d¢g d	¢gg d¢g d¢g d¢gƒdƒ d S )Nr   r(   r”   r•   r‘   r#  rÌ   re   r’   rµ   rL   rË   rÍ   r.  r5  r/  r)  rT  rV  rW  rñ   rX  ©r   r   r?   r?   r?   r@   Útest_label_ranking_lossÁ  s@     þ(ÿü(ÿür^  Úcsr_containerc                 C   s6   t t| t g d¢g d¢g¡ƒg d¢g d¢gƒdƒ d S )NrÌ   rÍ   rW  rX  re   )r   r   r,   rt   )r_  r?   r?   r@   Útest_label_ranking_loss_sparseì  s    &ÿür`  c                   C   s²  t  t¡�* tddgddggddgƒ W d   ƒ n1 s:0    Y  t  t¡�, tddgddggddggƒ W d   ƒ n1 s€0    Y  t  t¡�. tddgddggdgdggƒ W d   ƒ n1 sÈ0    Y  t  t¡�, tddggddgddggƒ W d   ƒ n1 �s0    Y  t  t¡�. tdgdggddgddggƒ W d   ƒ n1 �sZ0    Y  t  t¡�. tddgddggdgdggƒ W d   ƒ n1 �s¤0    Y  d S )Nr   r(   )rz   r{   r|   r   r?   r?   r?   r@   Ú$test_ranking_appropriate_input_shapeö  s    8:<<>ra  c                   C   sä   t tddggddggƒdƒ t tddggddggƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d¢gg d¢gƒdƒ t tg d	¢gg d¢gƒdƒ t tg d
¢gg d¢gƒdƒ d S )Nr(   r   re   r‘   r)  rÌ   r’   rµ   rË   rÍ   r]  r?   r?   r?   r@   Útest_ranking_loss_ties_handling  s    rb  c                  C   sH   t ddd�\} }| d }t||ƒ tj d¡ d¡\}}t||ƒ d S ©Nr   rú   )r'   rH  r(   )r$   rq   rú   )r   Ú_test_dcg_score_forr,   r0   r1   Úrandom_sample©ra   r>   r<   r?   r?   r@   Útest_dcg_score  s
    

rg  c                 C   s¾   t  t  | jd ¡d ¡}t| | ƒ}t| |ƒ}||k ¡ s>J ‚t| | dd�|k ¡ sXJ ‚|j| jd fksnJ ‚|j| jd fks„J ‚|t t  | ¡d d …d d d…f | j	dd�¡ksºJ ‚d S )Nr(   r$   r¯   ©Úkr   rB   ©Zaxis)
r,   Úlog2r-   r+   r   r  rz   rå   ÚsortrE   )r>   r<   ZdiscountÚidealrO   r?   r?   r@   rd    s    

rd  c               	   C   sN  t  t  d¡g¡} t  | j¡}t| |ƒ}t| |dd�}dt  t  dd¡¡ }|t | 	¡ |  
¡  g¡kslJ ‚|t || d d …d d d…f   	¡ g¡ksšJ ‚d|dd	d …f< t| |ƒ}t| |dd�}|t || d d …d d d…f   	¡ g¡ksðJ ‚|t |d d…  	¡ | dd	d …f  
¡  |dd …  	¡ | dd d	…f  
¡   g¡k�sJJ ‚d S )
Nr¯   T©Úignore_tiesr(   r$   rº   rB   r   rp   )r,   rî   r-   r�   r+   r   rk  rz   rå   rE   r  )r>   r<   ZdcgZdcg_ignore_tiesZ	discountsr?   r?   r@   Útest_dcg_ties%  s"    
 .
.""ÿÿÿrp  c               	   C   s>   t  d¡ d¡} t| | ddd�t t| | ddd�¡ks:J ‚d S )Né   )r$   r¹   rp   T)ri  ro  )r,   r-   rD   r   rz   rå   )rÂ   r?   r?   r@   Útest_ndcg_ignore_ties_with_k9  s    ÿrr  c                  C   s`   t  g d¢g¡} t  g d¢g¡}d}tjt|d�� t| |ƒ W d  ƒ n1 sR0    Y  dS )zDCheck `ndcg_score` exception when `y_true` contains negative values.)g{®Gázì¿gö(\�Âõà¿g®GázÞ¿gö(\�ÂõØ?gìQ¸…ëá?)gìQ¸…ë±?g×£p=
×Ó?r•   g…ëQ¸Õ?gHáz®GÑ?z7ndcg_score should not be used on negative y_true valuesr…   N)r,   rt   rz   r{   r|   r   )r>   r<   r�   r?   r?   r@   Ú test_ndcg_negative_ndarray_error@  s
    rs  c                  C   s�   t  d¡ dd¡} | t j d¡jdd| jd� }t| |ƒ}t| |dd	�}|t 	|¡ksZJ ‚|t 	d
¡kslJ ‚|d7 }t| |ƒt 	d
¡ksŒJ ‚d S )NéF   rº   rú   r   çš™™™™™É¿r£   rr   Trn  rL   rè   )
r,   r-   rD   r0   r1   r:  r+   r   rz   rå   )r>   r<   ZndcgZndcg_no_tiesr?   r?   r@   Útest_ndcg_invariantI  s    
rv  ro  c              
   C   s"  dt  d¡d d…  }t  t  ddd¡d¡}|t j d¡jdd	|jd
� }t||| d�t	 
dt  t  dd¡¡ ¡ksxJ ‚t||| d�t	 
dt  t  dd¡¡ ¡ks¦J ‚t||| d�t	 
dt  t  dd¡¡ ¡ksÔJ ‚t||d| d�t	 
dt  t  dd¡¡ ¡k�sJ ‚t||| d�t	 
dt  t  dd¡¡  ¡ ¡k�s:J ‚t||| d�t	 
dt  t  dd¡¡  ¡ ¡k�snJ ‚dt  d¡ }dt  t  dd¡¡  ¡ }t||| d�t	 
|t  d¡ ¡k�s¾J ‚t||| d�t	 
t  d¡¡k�sâJ ‚t||| d�t	 
|¡k�s J ‚t||| d�t	 
d¡k�sJ ‚d S )Nrp   rº   r¯   r¹   rB   )r¯   r(   r   ru  r£   rr   rn  r$   r(   rú   )Zlog_basero  )r¯   rº   é	   rL   )r,   ÚeyeZtiler-   r0   r1   r:  r+   r   rz   rå   rk  r   Úlog10r   r  r   r€   rE   )ro  r>   r<   Zy_score_noisyZexpected_dcg_scorer?   r?   r@   Útest_ndcg_toy_examplesT  s^    ÿÿþÿþÿþÿþ
ÿÿÿþ
ÿþ
ÿrz  c                  C   sH   d} t jt| d��" tdggdggƒ W d  ƒ n1 s:0    Y  dS )zhCheck that we raise an informative error message when trying to
    compute NDCG with a single document.zTComputing NDCG is only meaningful when there is more than 1 document. Got 1 instead.r…   r(   N)rz   r{   r|   r   )r  r?   r?   r@   Útest_ndcg_error_single_document{  s    ÿr{  c                  C   sH   t ddd�\} }| d }t||ƒ tj d¡ d¡\}}t||ƒ d S rc  )r   Ú_test_ndcg_score_forr,   r0   r1   re  rf  r?   r?   r@   Útest_ndcg_score†  s
    

r}  c                 C   sü   t | | ƒ}t | |ƒ}||k ¡ s$J ‚| dkjdd�}||  t t |  ¡ ¡¡ksXJ ‚|| t t | ¡ ¡¡ksxJ ‚||  t t| |ƒ|  t| | ƒ|   ¡ks¬J ‚|| t t | ¡ ¡¡ksÌJ ‚|j	| j	d fksâJ ‚|j	| j	d fksøJ ‚d S )Nr   r(   rj  )
r   r  rz   rå   r,   r€   rE   r�   r   r+   )r>   r<   rm  rO   Zall_zeror?   r?   r@   r|  Ž  s    

$ ÿÿ
 r|  c                  C   s€  t  g d¢¡} t| | dd�dks$J ‚t| | dd�dks:J ‚t t¡�" t| | dd�sXJ ‚W d   ƒ n1 sl0    Y  t t¡�" t| | dd�s”J ‚W d   ƒ n1 s¨0    Y  t t¡�" t| | dd�sÐJ ‚W d   ƒ n1 sä0    Y  t  g d¢¡}t| |dd�}t| |ƒ}||k�s"J ‚t| |d	d�d
k�s:J ‚tdd�\} }}t  ddd¡D ]"}tt| ||d�t	| ||ƒƒ �qXd S )Nrà   r(   )r^   gü©ñÒMbP?gš™™™™™¹¿gš™™™™™ñ?r   )r¨   r   r¨   g{®Gáz„?r­   re   Trh   g-Cëâ6?r¯   )
r,   rt   r   rz   r{   r|   rA   Zlinspacer   rf   )r>   rÕ   Zroc_auc_with_max_fpr_oneZunconstrained_roc_aucr=   ra   r^   r?   r?   r@   Útest_partial_roc_auc_scorež  s(    000

þr~  zy_true, k, true_scorer”   r$   rp   r•   c                 C   sF   t  g d¢g d¢g d¢g d¢g¡}t| ||d�}|t |¡ksBJ ‚d S )NrQ  ©r¨   r­   r®   r£   ©r®   r¨   r£   r­   ©r­   r£   r®   r¨   rh  ©r,   rt   r   rz   rå   )r>   ri  Ú
true_scorer<   rO   r?   r?   r@   Útest_top_k_accuracy_score¸  s    	üÿr„  zy_score, k, true_score)rB   rB   r(   r(   )rB   r(   rB   r(   )r£   r£   r¥   r¥   )r£   r¥   r£   r¥   c                 C   s€   g d¢}|   ¡ dkr$|  ¡ dkr$dnd}|dkr@| |k tj¡n|}t|| |d�}t||ƒ}||  krvt |¡ks|n J ‚d S )Nrà   r   r(   re   rh  )	ÚminÚmaxZastyper,   Zint64r   r   rz   rå   )r<   ri  rƒ  r>   rX   r=   rO   Z	score_accr?   r?   r@   Ú test_top_k_accuracy_score_binaryÍ  s     
r‡  zy_true, true_score, labels)r   r(   r(   r$   )r   r(   r(   r(   r  )rÂ   rò   rò   rÂ   )rÂ   rÃ   rÙ   rò   Úlabels_as_ndarrayc                 C   sV   |rt  |¡}t  g d¢g d¢g d¢g d¢g¡}t| |d|d�}|t |¡ksRJ ‚dS )z,Test when labels and y_score are multiclass.rQ  r  r€  r�  r$   ©ri  rÐ   N)r,   rî   rt   r   rz   rå   )r>   rƒ  rÐ   rˆ  r<   rO   r?   r?   r@   Ú0test_top_k_accuracy_score_multiclass_with_labelsä  s    
üÿ	rŠ  c                     s–   t jddddd�\‰ ‰tˆ ˆdd�\} }}}tdd�‰ˆ | |¡ t| |f||fƒD ]<\‰ ‰‡ ‡‡fdd„tddƒD ƒ}t t 	|¡dk¡sTJ ‚qTd S )	Nrú   rè   r   )rH  r7   Zn_informativer'   )r'   c                    s    g | ]}t ˆˆ ˆ ¡|d �‘qS )rh  )r   r3   )rˆ   ri  ©r5   r;   r6   r?   r@   rŠ     s   z8test_top_k_accuracy_score_increasing.<locals>.<listcomp>r$   )
r   Zmake_classificationr   r   r2   r  rM   r,   r  r³   )ZX_trainZX_testZy_trainZy_testZscoresr?   r‹  r@   Ú$test_top_k_accuracy_score_increasing  s    ÿ

ÿrŒ  c                 C   sB   t  g d¢g d¢g d¢g d¢g¡}t| ||d�t |¡ks>J ‚d S )N)r¯   r¯   rº   r   )r(   r¯   r¯   r¯   )r   r   rp   rp   r  rh  r‚  )r>   ri  rƒ  r<   r?   r?   r@   Útest_top_k_accuracy_score_ties  s    
üÿr�  z	y_true, kr¸   c                 C   sr   t  g d¢g d¢g d¢g d¢g¡}d}tjt|d�� t| ||d�}W d   ƒ n1 sX0    Y  |dksnJ ‚d S )	NrQ  )r¨   r®   r­   r£   )r£   r¨   r®   r­   )r­   r£   r¨   r®   zu'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a perfect score and is therefore meaningless.r…   rh  r(   )r,   rt   rz   rŒ   r   r   )r>   ri  r<   r�   rO   r?   r?   r@   Ú!test_top_k_accuracy_score_warning)  s    üÿ	ÿ,rŽ  zy_true, y_score, labels, msg)r   g=
×£p=â?r(   r$   )r£   r¨   r¥   )r®   r­   r­   rÆ   )r®   re   r¨   z9y type must be 'binary' or 'multiclass', got 'continuous'zZNumber of classes in 'y_true' \(4\) not equal to the number of classes in 'y_score' \(3\).)rÄ   rÄ   rÂ   rÃ   z"Parameter 'labels' must be unique.z#Parameter 'labels' must be ordered.)r   r   r(   r$   zSNumber of given labels \(4\) not equal to the number of classes in 'y_score' \(3\).rV  z3'y_true' contains labels not in parameter 'labels'.)re   r£   r£   )r­   r®   r£   z}`y_true` is binary while y_score is 2d with 3 classes. If `y_true` does not contain all the labels, `labels` must be providedc                 C   sB   t jt|d��  t| |d|d� W d   ƒ n1 s40    Y  d S )Nr…   r$   r‰  )rz   r{   r|   r   )r>   r<   rÐ   ró   r?   r?   r@   Útest_top_k_accuracy_score_errorB  s    Sr�  c                 C   sJ   | g d¢g d¢gƒ}t  g d¢g d¢g¡}t||ƒ}|t d¡ksFJ ‚d S )Nrµ   r‘   )re   r~   r¤   r6  )r,   rt   r   rz   rå   )r_  r>   r<   Úresultr?   r?   r@   ÚOtest_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_input™  s    
r‘  Úmetricrí   ))FT)r   r(   )r“   rL   )ÚzeroÚonec                 C   s    t j d¡}d|d  }}|j||dd�}| |¡}| |||d�}t|tƒr\t  |¡rœJ ‚n@|\}}	}
t  |¡ ¡ rxJ ‚t  |	¡ ¡ rŠJ ‚t  |
¡ ¡ rœJ ‚dS )zÀCheck that the metric works with different types of `pos_label`.

    We can expect `pos_label` to be a bool, an integer, a float, a string.
    No error should be raised for those types.
    é*   rú   rB   T)rs   Úreplacer  N)	r,   r0   r1   Úchoicer÷   Ú
isinstancerF   rŸ   Úany)r’  rí   r:   r7   rH   r>   Zy_probar�  Zmetric_1Zmetric_2rl   r?   r?   r@   Ú#test_ranking_metric_pos_label_types¥  s    


rš  c                 C   sL   t j | ¡}|jdddd�}| d¡}t||ƒ\}}}t  |d ¡sHJ ‚dS )z¶Check that thresholds do not exceed 1.0 when `y_score` is a probability
    estimate.

    Non-regression test for:
    https://github.com/scikit-learn/scikit-learn/issues/26193
    r   r$   rú   rr   N)r,   r0   r1   ru   r÷   r   Úisinf)rï   r:   r>   r<   ra   rl   r?   r?   r@   Ú(test_roc_curve_with_probablity_estimates¿  s
    
rœ  c                  C   s’   t dd�\} }}d}tjt|d�� t| |d� W d  ƒ n1 sD0    Y  d}tjt|d�� t| ||d� W d  ƒ n1 s„0    Y  dS )	z)Check the message for future deprecation.Trh   z)probas_pred was deprecated in version 1.5r…   )Úprobas_predNz4`probas_pred` and `y_score` cannot be both specified)r�  r<   )rA   rz   rŒ   ÚFutureWarningr   r{   r|   )r>   ra   r<   Zwarn_msgÚ	error_msgr?   r?   r@   Ú/test_precision_recall_curve_deprecation_warningÎ  s    þ$ýr   )NF)r¯   rö   r   )�rž   r½   Únumpyr,   rz   Zscipyr   Zsklearnr   r   Zsklearn.datasetsr   Zsklearn.exceptionsr   Zsklearn.linear_modelr   Zsklearn.metricsr   r	   r
   r   r   r   r   r   r   r   r   r   r   Zsklearn.metrics._rankingr   r   Zsklearn.model_selectionr   Zsklearn.preprocessingr   Zsklearn.random_projectionr   Zsklearn.utils._testingr   r   r   r   r   Zsklearn.utils.extmathr   Zsklearn.utils.fixesr    Zsklearn.utils.validationr!   r"   r#   ZCURVE_FUNCSrA   rK   rS   rY   rf   ÚmarkZparametrizern   rv   ry   r}   r   rƒ   r�   r    r«   r´   r·   rÀ   rt   rØ   rÝ   râ   ræ   rð   rô   rù   rý   rÿ   r  Úfilterwarningsr  r  r  r
  r  r  r  r  r   r!  r"  r(  r*  r+  r,  r-  r2  r3  r9  r<  r=  r?  rC  rG  rK  rN  rO  rM   rP  rS  rZ  r[  r\  r^  r`  ra  rb  rg  rd  rp  rr  rs  rv  rz  r{  r}  r|  r~  r„  r‡  rŠ  rŒ  r�  rŽ  r�  r‘  rš  rœ  r   r?   r?   r?   r@   Ú<module>   s  <
ý0
r
üþ	
2þþ

üþ	
"þ
"ýú	ýú
ú
ú
ú	ýýýÂþG	û	û	û	úâþ#

	




 $"	ìþ
øþ


ûþ

ýûþ
;'ÿ
 üþ	7ýþ
+
		
&
ýþúþüþ	ýþ
þþü÷ü÷ü÷ü÷ü÷ü÷û»þRÿÿ