a
    Žù»j”Y  ã                   @   sð  d dl mZmZmZ d dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlZddlmZmZmZmZ ddlmZ dd	lmZmZ d
dlmZmZ d
dlmZmZmZ e d¡Ze d¡Zd dlm Z m!Z!m"Z" d dl#mZ d dl$m%Z%m&Z&m'Z' ej(ddd�dd„ ƒZ)dd„ Z*didd„Z+dd„ Z,dd„ Z-d d!„ Z.ej/ 0d"dd#g¡ej/ 0d$eeg¡d%d&„ ƒƒZ1eej/ 0d"dd#g¡ej/ 0d'd(d)g¡ej/ 0d$eeg¡d*d+„ ƒƒƒƒZ2eej/ 0d$eeg¡d,d-„ ƒƒZ3d.d/„ Z4d0d1„ Z5ej/ 0d2dd3g¡d4d5„ ƒZ6d6d7„ Z7G d8d9„ d9e8ƒZ9djd:d;„Z:d<d=„ Z;d>d?„ Z<ej/ 0d@e=e>dƒƒ¡dAdB„ ƒZ?dCdD„ Z@dEdF„ ZAdGdH„ ZBdIdJ„ ZCdKdL„ ZDdMdN„ ZEdOdP„ ZFdQdR„ ZGdSdT„ ZHdUdV„ ZIdWdX„ ZJej/ 0dYdZd[g¡ej/jKejLd\k�o¸ejLd]kd^d_�d`da„ ƒƒZMdbdc„ ZNej/ 0dddedfg¡dgdh„ ƒZOdS )ké    )Úabsolute_importÚdivisionÚprint_functionN©Úrandom)Úsleep)Úuuid4é   )ÚParallelÚdelayedÚparallel_backendÚparallel_config)ÚDaskDistributedBackend)ÚAutoBatchingMixinÚThreadingBackendé   )ÚnpÚ
with_numpy)Ú_recursive_backend_infoÚ_test_deadlock_with_generatorÚ8_test_parallel_unordered_generator_returns_fastest_firstÚdistributedÚdask)ÚClientÚLocalClusterÚ
get_client)Útime)ÚcleanupÚclusterÚincÚfunctionT)ÚscopeZautousec                 c   sZ   ddl m} dd„ |jD ƒ}d V  | ¡ D ]*\}}|d u rJtj |d ¡ q*|tj|< q*d S )Nr   )ÚParallelBackendBasec                 S   s   i | ]}|t j |¡“qS © )ÚosÚenvironÚget)Ú.0Úkr#   r#   ú;E:\soccer\ml_env\Lib\site-packages\joblib/test/test_dask.pyÚ
<dictcomp>(   ó    z(avoid_dask_env_leaks.<locals>.<dictcomp>)Zjoblib._parallel_backendsr"   ZMAX_NUM_THREADS_VARSÚitemsr$   r%   Úpop)Ztmp_pathr"   Ú	old_valuer(   Úvr#   r#   r)   Úavoid_dask_env_leaks!   s    r0   c                  O   s   d S ©Nr#   )ÚargsÚkwargsr#   r#   r)   Únoop3   s    r4   çš™™™™™©?c                 C   s   t |ƒ | rtdƒ‚d S )Nzcondition evaluated to True)r   Ú
ValueError)Ú	conditionÚdurationr#   r#   r)   Úslow_raise_value_error7   s    r9   c                    sF   |  dd„ ¡}i }| ¡ D ]&\}}t‡ fdd„t|ƒD ƒƒ||< q|S )Nc                 S   s   | j S r1   )Úlog)Zdask_workerr#   r#   r)   Ú<lambda>>   r+   zcount_events.<locals>.<lambda>c                    s   g | ]}|d  ˆ kr|‘qS ©r   r#   )r'   Úevent©Ú
event_namer#   r)   Ú
<listcomp>B   r+   z count_events.<locals>.<listcomp>)Úrunr,   ÚlenÚlist)r?   ÚclientZworker_eventsZevent_countsÚwÚeventsr#   r>   r)   Úcount_events=   s    ÿ
rG   c                 C   s8  t ƒ ��\}\}}t|d | d��à}tdd��¶ tƒ dd„ tdƒD ƒƒ}|dd	„ tdƒD ƒksbJ ‚t t¡�( tƒ d
d„ tdƒD ƒƒ W d   ƒ n1 sš0    Y  tƒ dd„ tdƒD ƒƒ}|dd	„ tdƒD ƒksÖJ ‚W d   ƒ n1 sê0    Y  W d   ƒ n1 �s
0    Y  W d   ƒ n1 �s*0    Y  d S )NÚaddress©Úloopr   ©Úbackendc                 s   s   | ]}t tƒ|ƒV  qd S r1   ©r   r   ©r'   Úir#   r#   r)   Ú	<genexpr>K   r+   ztest_simple.<locals>.<genexpr>é
   c                 S   s   g | ]}t |ƒ‘qS r#   ©r   rN   r#   r#   r)   r@   L   r+   ztest_simple.<locals>.<listcomp>c                 s   s   | ]}t tƒ|d kƒV  qdS )é   N)r   r9   rN   r#   r#   r)   rP   O   s   c                 s   s   | ]}t tƒ|ƒV  qd S r1   rM   rN   r#   r#   r)   rP   S   r+   c                 S   s   g | ]}t |ƒ‘qS r#   rR   rN   r#   r#   r)   r@   T   r+   )r   r   r   r
   ÚrangeÚpytestÚraisesr6   ©rJ   ÚsÚaÚbrD   Úseqr#   r#   r)   Útest_simpleG   s    
ÿ&r\   c                 C   s  t jtju sJ ‚tƒ �ð\}\}}t|d | d��¸}tdd��Ž tƒ �h}|j}t|t ƒsZJ ‚|j	|u shJ ‚|j
dksvJ ‚|dd„ ttdƒƒD ƒƒ |j
d	ksžJ ‚W d   ƒ n1 s²0    Y  W d   ƒ n1 sÐ0    Y  W d   ƒ n1 sî0    Y  W d   ƒ n1 �s0    Y  d S )
NrH   rI   r   rK   r   c                 s   s   | ]}t d d„ ƒƒ V  qdS )c                   S   s   d S r1   r#   r#   r#   r#   r)   r;   j   r+   z?test_dask_backend_uses_autobatching.<locals>.<genexpr>.<lambda>N©r   ©r'   Ú_r#   r#   r)   rP   j   r+   z6test_dask_backend_uses_autobatching.<locals>.<genexpr>ç     ˆÃ@rQ   )r   Zcompute_batch_sizer   r   r   r   r
   Ú_backendÚ
isinstanceÚparallelZ_effective_batch_sizerT   Úint)rJ   rX   rY   rZ   rD   rc   rL   r#   r#   r)   Ú#test_dask_backend_uses_autobatchingW   s    ÿÿre   Ún_jobséÿÿÿÿÚcontextc              	   C   sd   t jddd��B |dƒ� td | ƒ W d   ƒ n1 s80    Y  W d   ƒ n1 sV0    Y  d S ©Nr	   ©Ú	n_workersÚthreads_per_workerr   )r   r   r   )rf   rh   r#   r#   r)   ÚAtest_parallel_unordered_generator_returns_fastest_first_with_daskn   s    rm   Ú	return_asÚ	generatorZgenerator_unorderedc              	   C   sf   t jddd��D | dƒ� td ||ƒ W d   ƒ n1 s:0    Y  W d   ƒ n1 sX0    Y  d S ri   )r   r   r   )rh   rn   rf   r#   r#   r)   Ú%test_deadlock_with_generator_and_dasku   s    rp   c              
   C   s   t jddd��Þ tjtdƒtjd�}tdƒD ]\}| dƒ� t|d�}W d   ƒ n1 sX0    Y  t|ƒdksrJ ‚t	dd	„ |D ƒƒs,J ‚q,| dƒ� tƒ }W d   ƒ n1 s®0    Y  t|ƒdksÈJ ‚t	d
d	„ |D ƒƒsÞJ ‚W d   ƒ n1 sò0    Y  d S )Nr	   rj   g    ÐcA©Zdtyper   ©Údataé   c                 s   s   | ]\}}|d kV  qdS ©r   Nr#   ©r'   Únamer_   r#   r#   r)   rP   ˆ   s   z4test_nested_parallelism_with_dask.<locals>.<genexpr>c                 s   s   | ]\}}|d kV  qdS ru   r#   rv   r#   r#   r)   rP   �   s   )
r   r   r   Úonesrd   Úuint8rT   r   rB   Úall)rh   rs   rO   Zbackend_types_and_levelsr#   r#   r)   Ú!test_nested_parallelism_with_dask~   s    
(ÿ
$ÿr{   c                   C   s   t ƒ S r1   r   r#   r#   r#   r)   Úrandom2•   s    r|   c              
   C   s´   t ƒ �š\}\}}t|d | d��b}tdd��8 tƒ dd„ tdƒD ƒƒ\}}||ksVJ ‚W d   ƒ n1 sj0    Y  W d   ƒ n1 sˆ0    Y  W d   ƒ n1 s¦0    Y  d S )NrH   rI   r   rK   c                 s   s   | ]}t tƒƒ V  qd S r1   )r   r|   rN   r#   r#   r)   rP   �   r+   z3test_dont_assume_function_purity.<locals>.<genexpr>r	   ©r   r   r   r
   rT   )rJ   rX   rY   rZ   rD   ÚxÚyr#   r#   r)   Ú test_dont_assume_function_purity™   s
    r€   ÚmixedFc              
   C   s$  ddl m} |s(dd„ tdƒD ƒ}d}ndd„ tdƒD ƒ}d}t||ƒƒ|ksRJ ‚tƒ �¶\}\}}t|d	 | d
��~}tdd��  tddd�|ƒ}	W d   ƒ n1 s¤0    Y  dd„ }
| dd¡}| 	|
¡}t
dd„ |D ƒƒsâJ ‚W d   ƒ n1 sö0    Y  W d   ƒ n1 �s0    Y  d S )Nr   )ÚBatchc                 S   s   g | ]}t tƒ|ƒ‘qS r#   rM   rN   r#   r#   r)   r@   ¦   r+   z&test_dask_funcname.<locals>.<listcomp>rt   Zbatch_of_inc_4_callsc                 S   s,   g | ]$}|d  rt tƒ|ƒn
t tƒ|ƒ‘qS )r	   )r   Úabsr   rN   r#   r#   r)   r@   ©   r+   Zmixed_batch_of_inc_4_callsrH   rI   r   rK   r	   rz   )Ú
batch_sizeZpre_dispatchc                 S   s
   t | jƒS r1   )rC   Ztransition_log)Zdask_schedulerr#   r#   r)   Úf³   s    ztest_dask_funcname.<locals>.fÚ4Ú2c                 s   s   | ]}d |d v V  qdS )Zbatch_of_incr   Nr#   )r'   Útupr#   r#   r)   rP   ¸   r+   z%test_dask_funcname.<locals>.<genexpr>)Zjoblib._daskr‚   rT   Úreprr   r   r   r
   ÚreplaceZrun_on_schedulerrz   )rJ   r�   r‚   ÚtasksZ
batch_reprrX   rY   rZ   rD   r_   r…   r:   r#   r#   r)   Útest_dask_funcname¡   s    .
rŒ   c               	      sš  dd„ t dƒD ƒ} t d¡‰ˆ tdƒ¡‰ d‡ ‡fdd„	‰tdd	d
�}t|ƒ}�z*tdd��( tƒ ‡fdd„| D ƒƒ}W d   ƒ n1 sŠ0    Y  | dd„ t dƒD ƒks®J ‚t	d|ƒ}t
| ¡ ƒdksÌJ ‚tdd„ |D ƒƒsâJ ‚tdd��* tƒ ‡ ‡fdd„| D ƒƒ}W d   ƒ n1 �s0    Y  t	d|ƒ}t
| ¡ ƒdk�sHJ ‚tdd„ |D ƒƒ�s`J ‚W |jdd� |jdd� n|jdd� |jdd� 0 d S )Nc                 S   s   g | ]}g ‘qS r#   r#   r^   r#   r#   r)   r@   Æ   r+   z;test_no_undesired_distributed_cache_hit.<locals>.<listcomp>éd   Únumpyg    €„.Ac                    s(   |d urˆj  |ˆ ¡ |  tƒ j¡ | S r1   )ZtestingZassert_array_equalÚappendr   Úhex)Úlist_rs   )ÚXr   r#   r)   Úisolated_operationÊ   s    zCtest_no_undesired_distributed_cache_hit.<locals>.isolated_operationr   r	   rj   r   rK   c                 3   s   | ]}t ˆ ƒ|ƒV  qd S r1   r]   ©r'   r‘   )r“   r#   r)   rP   Õ   r+   z:test_no_undesired_distributed_cache_hit.<locals>.<genexpr>c                 S   s   g | ]}g ‘qS r#   r#   r^   r#   r#   r)   r@   Ù   r+   úreceive-from-scatterr   c                 S   s   g | ]}t |ƒd k‘qS r<   ©rB   ©r'   Úrr#   r#   r)   r@   à   r+   c                 3   s   | ]}t ˆƒ|ˆ d �V  qdS )rr   Nr]   r”   )r’   r“   r#   r)   rP   å   s   c                 S   s   g | ]}t |ƒd k‘qS r<   r–   r—   r#   r#   r)   r@   ì   r+   é   ©Útimeout)N)rT   rU   ÚimportorskipZarangerd   r   r   r   r
   rG   ÚsumÚvaluesrz   Úclose)Úlistsr   rD   ÚresÚcountsr#   )r’   r“   r   r)   Ú'test_no_undesired_distributed_cache_hit»   s0    
6
ÿ(
ÿr£   c                   @   s(   e Zd Zdd„ Zdd„ ZeZdd„ ZdS )ÚCountSerializedc                 C   s   || _ d| _d S )Nr   )r~   Úcount)Úselfr~   r#   r#   r)   Ú__init__ó   s    zCountSerialized.__init__c                 C   s   | j t|d|ƒ S )Nr~   )r~   Úgetattr)r¦   Úotherr#   r#   r)   Ú__add__÷   s    zCountSerialized.__add__c                 C   s   |  j d7  _ t| jffS )Nr   )r¥   r¤   r~   )r¦   r#   r#   r)   Ú
__reduce__ü   s    zCountSerialized.__reduce__N)Ú__name__Ú
__module__Ú__qualname__r§   rª   Ú__radd__r«   r#   r#   r#   r)   r¤   ò   s   r¤   c                 C   s   | | | | | S r1   r#   )rY   rZ   ÚcÚdÚer#   r#   r)   Úadd5  s    r³   c                    s   dd„ t dƒD ƒ\}‰‰‰ttƒ‰‡‡‡‡fdd„t dƒD ƒ}|ˆˆˆˆddd�ˆˆˆˆˆdd�ˆˆˆˆˆˆd�g7 }d	d„ |D ƒ}tƒ �î\}}t|d
 | d��¸‰ td|ˆˆgd��* tdd�|ƒ}||ksÐJ ‚W d   ƒ n1 sä0    Y  t t	¡�@ td| dd�� W d   ƒ n1 �s 0    Y  W d   ƒ n1 �s@0    Y  W d   ƒ n1 �s`0    Y  W d   ƒ n1 �s€0    Y  |j
}ˆj
|k�s J ‚ˆj
|k�s°J ‚ˆj
}|ˆˆˆfD ]}	d|	_
�qÂtƒ �–\}}t|d
 | d��`‰ tƒ ‰|ˆˆfD ]}
ˆ j|
dd�ˆt|
ƒ< �qþ‡ ‡fdd„|D ƒ}||k�s>J ‚W d   ƒ n1 �sT0    Y  W d   ƒ n1 �st0    Y  |j
}ˆj
|k�s”J ‚ˆj
|k�s¤J ‚||k�s²J ‚tdd„ tj d¡D ƒƒ}|dk �rìˆj
|d k�süJ ‚nˆj
|k�süJ ‚d S )Nc                 s   s   | ]}t |ƒV  qd S r1   )r¤   rN   r#   r#   r)   rP     r+   z&test_manual_scatter.<locals>.<genexpr>rt   c              	      s   g | ]}ˆ ˆˆˆd dd�‘qS )rt   é   ©r±   r²   r#   r^   )r…   r~   r   Úzr#   r)   r@     r+   z'test_manual_scatter.<locals>.<listcomp>rQ   r´   rµ   c                 S   s    g | ]\}}}||i |¤Ž‘qS r#   r#   ©r'   Úfuncr2   r3   r#   r#   r)   r@     r+   rH   rI   r   )rL   Úscatterr   )r„   )rL   rJ   r¹   r   T)Ú	broadcastc              	      s`   g | ]X\}}}ˆ j |g‡fd d„|D ƒ¢R i t‡fdd„| ¡ D ƒƒ¤dttƒ ƒi¤Ž ¡ ‘qS )c                 3   s   | ]}ˆ   t|ƒ|¡V  qd S r1   ©r&   Úid)r'   Úarg©Ú	scatteredr#   r)   rP   :  r+   z1test_manual_scatter.<locals>.<listcomp>.<genexpr>c                 3   s&   | ]\}}|ˆ   t|ƒ|¡fV  qd S r1   r»   )r'   ÚkeyÚvaluer¾   r#   r)   rP   ;  s   ÿrÀ   )ÚsubmitÚdictr,   Ústrr   Úresultr·   )rD   r¿   r#   r)   r@   7  s   
÷ÿþþýùc                 s   s   | ]}t |ƒV  qd S r1   )rd   )r'   r/   r#   r#   r)   rP   M  r+   Ú.)iç  rt   )rT   r   r³   r   r   r   r
   rU   rV   Ú	TypeErrorr¥   rÃ   r¹   r¼   Útupler   Ú__version__Úsplit)rJ   rE   r‹   ÚexpectedrX   r_   Zresults_parallelZ%n_serialization_scatter_with_parallelZn_serialization_with_parallelÚvarÚobjZresults_nativeZn_serialization_scatter_nativeZdistributed_versionr#   )rD   r…   r¿   r~   r   r¶   r)   Útest_manual_scatter  sP    
ý*€

öN
rÎ   c           	   
      sö  t  d¡}|jtdƒ|jd�‰ |jtdƒ|jd�}ˆ gd |gd  }tƒ �´\}\}}t|d | d��|}tdd��( tƒ d	d
„ t	|ƒD ƒƒ W d   ƒ n1 s 0    Y  t
d|ƒ}||d  ||d   dksÔJ ‚W d   ƒ n1 sè0    Y  W d   ƒ n1 �s0    Y  tƒ �È\}\}}t|d | d��Ž}tdd��, tƒ ‡ fdd
„tdƒD ƒƒ W d   ƒ n1 �sr0    Y  t
d|ƒ}||d  dk�sœJ ‚||d  dk�s²J ‚W d   ƒ n1 �sÈ0    Y  W d   ƒ n1 �sè0    Y  d S )NrŽ   r`   rq   rS   rH   rI   r   rK   c                 s   s&   | ]\}}t tƒ||||d �V  qdS ))ÚoptN©r   r4   )r'   rO   rs   r#   r#   r)   rP   f  s   ÿz$test_auto_scatter.<locals>.<genexpr>r•   r	   c                 3   s$   | ]}t tƒˆ d d… |ƒV  qd S )NrS   rÐ   rN   ©Zdata1r#   r)   rP   s  r+   r´   r   )rU   rœ   rx   rd   ry   r   r   r   r
   Ú	enumeraterG   rT   )	Úloop_in_threadr   Zdata2Zdata_to_processrX   rY   rZ   rD   r¢   r#   rÑ   r)   Útest_auto_scatter[  s&    

þ&
^<
rÔ   Úretry_noc              
      sà   t  d¡‰d‰ d}‡fdd„‰‡ ‡fdd„‰tƒ �š\}\}}t|d | d��b}td	d
��8 ˆ d¡‰tƒ ‡‡fdd„t|ƒD ƒƒ}W d   ƒ n1 s–0    Y  W d   ƒ n1 s´0    Y  W d   ƒ n1 sÒ0    Y  d S )NrŽ   rQ   c                    s
   ˆ   | ¡S r1   )r�   )r~   rO   Új)r   r#   r)   Úmy_sum‚  s    z#test_nested_scatter.<locals>.my_sumc                    sX   t ƒ }tdd��0 tƒ ‡ ‡‡fdd„tˆƒD ƒƒ}W d   ƒ n1 sF0    Y  t|ƒS )Nr   rK   c                 3   s&   | ]}t ˆƒˆ |d … ˆ|ƒV  qd S r1   r]   )r'   rÖ   )ÚarrayrO   r×   r#   r)   rP   ˆ  s   zEtest_nested_scatter.<locals>.outer_function_joblib.<locals>.<genexpr>)r   r   r
   rT   r�   )rØ   rO   rD   Úresults)ÚNUM_INNER_TASKSr×   )rØ   rO   r)   Úouter_function_joblib…  s    ÿ&z2test_nested_scatter.<locals>.outer_function_joblibrH   rI   r   rK   i'  c                 3   s$   | ]}t ˆƒˆ |d … |ƒV  qd S r1   r]   rN   )Úmy_arrayrÛ   r#   r)   rP   ‘  s   ÿz&test_nested_scatter.<locals>.<genexpr>)rU   rœ   r   r   r   rx   r
   rT   )rJ   rÕ   ZNUM_OUTER_TASKSrX   rY   rZ   r_   r#   )rÚ   rÜ   r×   r   rÛ   r)   Útest_nested_scatter{  s    

þrÝ   c              
      sr  dd„ ‰ t ƒ ��L\}\}}t|d | d��x}tdd��N tdd�‡ fd	d
„tdƒD ƒƒ}|D ]}tt|ƒƒdks\J ‚q\W d   ƒ n1 sŠ0    Y  W d   ƒ n1 s¨0    Y  t|d | d��z}tdd��N tdd�‡ fdd
„tdƒD ƒƒ}|D ]}tt|ƒƒdksôJ ‚qôW d   ƒ n1 �s$0    Y  W d   ƒ n1 �sD0    Y  W d   ƒ n1 �sd0    Y  d S )Nc                  S   sH   t tdd�dd„ tdƒD ƒƒƒ} | t tdd�dd„ tdƒD ƒƒƒO } | S )Nr	   ©rf   c                 s   s   | ]}t tjƒƒ V  qd S r1   ©r   r$   Úgetpidr^   r#   r#   r)   rP   ™  r+   zOtest_nested_backend_context_manager.<locals>.get_nested_pids.<locals>.<genexpr>c                 s   s   | ]}t tjƒƒ V  qd S r1   rß   r^   r#   r#   r)   rP   š  r+   )Úsetr
   rT   )Zpidsr#   r#   r)   Úget_nested_pids˜  s     $z<test_nested_backend_context_manager.<locals>.get_nested_pidsrH   rI   r   rK   r	   rÞ   c                 3   s   | ]}t ˆ ƒƒ V  qd S r1   r]   r^   ©râ   r#   r)   rP      s   z6test_nested_backend_context_manager.<locals>.<genexpr>rQ   c                 3   s   | ]}t ˆ ƒƒ V  qd S r1   r]   r^   rã   r#   r)   rP   ©  s   )r   r   r   r
   rT   rB   rá   )rÓ   rX   rY   rZ   rD   Z
pid_groupsZ	pid_groupr#   rã   r)   Ú#test_nested_backend_context_manager—  s     ÿRÿrä   c           	         s&  dd„ ‰ ‡ fdd„‰t ƒ �ö\}\}}t|d | d��¾}tdd��” tƒ �H}ˆ |ƒd	ksZJ ‚|jd
kshJ ‚|‡fdd„tdƒD ƒƒ}W d   ƒ n1 s–0    Y  |D ] \}}|d	ks¸J ‚|d
ks¤J ‚q¤W d   ƒ n1 sÚ0    Y  W d   ƒ n1 sø0    Y  W d   ƒ n1 �s0    Y  d S )Nc                 S   s
   | j jjS r1   )ra   Ú	__class__r¬   ©Úpr#   r#   r)   Ú_backend_type´  s    zJtest_nested_backend_context_manager_implicit_n_jobs.<locals>._backend_typec                     s8   t ƒ �} ˆ | ƒ| jfW  d   ƒ S 1 s*0    Y  d S r1   )r
   rf   ræ   )rè   r#   r)   Úget_nested_implicit_n_jobs·  s    zWtest_nested_backend_context_manager_implicit_n_jobs.<locals>.get_nested_implicit_n_jobsrH   rI   r   rK   r   rg   c                 3   s   | ]}t ˆ ƒƒ V  qd S r1   r]   r^   )ré   r#   r)   rP   Á  s   zFtest_nested_backend_context_manager_implicit_n_jobs.<locals>.<genexpr>r	   )r   r   r   r
   rf   rT   )	rJ   rX   rY   rZ   rD   rç   Zall_nested_n_jobsZbackend_typeZnested_n_jobsr#   )rè   ré   r)   Ú3test_nested_backend_context_manager_implicit_n_jobs°  s    ÿ&rê   c              	   C   sn   t  t¡�:}tdd�� W d   ƒ n1 s,0    Y  W d   ƒ n1 sJ0    Y  dt|jƒ ¡ v sjJ ‚d S )Nr   rK   zcreate a dask client)rU   rV   r6   r   rÄ   rÁ   Úlower)rJ   Úinfor#   r#   r)   Útest_errorsÉ  s    <rí   c              
   C   s&  t ƒ ��\}\}}t|d | d��Ð}tdd��F tdd�dd„ td	ƒD ƒƒ}t|d
 d
 d
 tƒsfJ ‚W d   ƒ n1 sz0    Y  tdd��F tdd�dd„ td	ƒD ƒƒ}t|d
 d
 d
 tƒsÆJ ‚W d   ƒ n1 sÚ0    Y  W d   ƒ n1 sø0    Y  W d   ƒ n1 �s0    Y  d S )NrH   rI   r   rK   r	   rÞ   c                 s   s   | ]}t tƒd d�V  qd S )N©Znested_require©r   Úouterr^   r#   r#   r)   rP   Ö  s   z.test_correct_nested_backend.<locals>.<genexpr>r   r   c                 s   s   | ]}t tƒd d�V  qdS )Z	sharedmemrî   Nrï   r^   r#   r#   r)   rP   Ý  s   )r   r   r   r
   rT   rb   r   r   )rJ   rX   rY   rZ   rD   rÅ   r#   r#   r)   Útest_correct_nested_backendÑ  s    ÿ8ÿrñ   c                    s"   t ddd�‡ fdd„tdƒD ƒƒS )Nr	   Úthreads)rf   Zpreferc                 3   s   | ]}t tƒˆ ƒV  qd S r1   )r   Úmiddler^   rî   r#   r)   rP   ä  s   zouter.<locals>.<genexpr>r   ©r
   rT   rî   r#   rî   r)   rð   ã  s    ÿrð   c                 C   s   t d| d�dd„ tdƒD ƒƒS )Nr	   )rf   Úrequirec                 s   s   | ]}t tƒƒ V  qd S r1   )r   Úinnerr^   r#   r#   r)   rP   ê  r+   zmiddle.<locals>.<genexpr>r   rô   )rõ   r#   r#   r)   ró   é  s    ró   c                   C   s   t ƒ jS r1   )r
   ra   r#   r#   r#   r)   rö   í  s    rö   c              	   C   sx   t | ddd��V tdd��, tdd�dd	„ td
ƒD ƒƒ W d   ƒ n1 sL0    Y  W d   ƒ n1 sj0    Y  d S )NFT)rJ   Ú	processesZset_as_defaultr   rK   rt   rÞ   c                 s   s   | ]}t tƒ|ƒV  qd S r1   )r   r¼   rN   r#   r#   r)   rP   õ  r+   z0test_secede_with_no_processes.<locals>.<genexpr>r	   )r   r   r
   rT   rI   r#   r#   r)   Útest_secede_with_no_processesñ  s    rø   c                 C   s   ddl m} |ƒ jS )Nr   )Ú
get_worker)r   rù   rH   )r_   rù   r#   r#   r)   Ú_worker_addressø  s    rú   c              
   C   s"  t ƒ ��\}\}}t|d | d��Ì}td|d d��> tƒ dd„ tdƒD ƒƒ}||d gd ksdJ ‚W d   ƒ n1 sx0    Y  td|d d��> tƒ dd„ tdƒD ƒƒ}||d gd ksÂJ ‚W d   ƒ n1 sÖ0    Y  W d   ƒ n1 sô0    Y  W d   ƒ n1 �s0    Y  d S )	NrH   rI   r   )rL   Úworkersc                 s   s   | ]}t tƒ|ƒV  qd S r1   ©r   rú   rN   r#   r#   r)   rP     r+   z-test_dask_backend_keywords.<locals>.<genexpr>rQ   c                 s   s   | ]}t tƒ|ƒV  qd S r1   rü   rN   r#   r#   r)   rP     r+   r}   rW   r#   r#   r)   Útest_dask_backend_keywordsþ  s    4rý   c              	   C   s¨   t d| d��ˆ}tdd��( tƒ dd„ tdƒD ƒƒ W d   ƒ n1 sF0    Y  tƒ }|jjjr|tdƒ tƒ |d	 k sVJ ‚qV|j	r†J ‚W d   ƒ n1 sš0    Y  d S )
NF)r÷   rJ   r   rK   c                 s   s   | ]}t tƒ|ƒV  qd S r1   rM   rN   r#   r#   r)   rP     r+   z/test_scheduler_tasks_cleanup.<locals>.<genexpr>rQ   g{®Gáz„?r´   )
r   r   r
   rT   r   r   Z	schedulerr‹   r   Zfutures)rJ   rD   Ústartr#   r#   r)   Útest_scheduler_tasks_cleanup
  s    6
rÿ   Úcluster_strategyÚadaptiveÚlate_scalingz2.1.1z1.28.0z?distributed bug - https://github.com/dask/distributed/pull/2841)Úreasonc                 C   s®   t dddd�}t|ƒ}| dkr.|jddd� n| dkr@| d¡ zVtdd	��( tƒ d
d„ tdƒD ƒƒ W d   ƒ n1 sz0    Y  W | ¡  | ¡  n| ¡  | ¡  0 d S )Nr   Fr	   ©rk   r÷   rl   r  )ÚminimumÚmaximumr  r   rK   c                 s   s   | ]}t tƒ|ƒV  qd S r1   rM   rN   r#   r#   r)   rP   +  r+   z(test_wait_for_workers.<locals>.<genexpr>rQ   )r   r   ZadaptÚscaler   r
   rT   rŸ   )r   r   rD   r#   r#   r)   Útest_wait_for_workers  s    
8
ÿr  c               	   C   s0  t dddd�} t| ƒ}�z tddd��Z d}tjt|d	��( tƒ d
d„ tdƒD ƒƒ W d   ƒ n1 sh0    Y  W d   ƒ n1 s†0    Y  tddd��Z d}tjt|d	��( tƒ dd„ tdƒD ƒƒ W d   ƒ n1 sÞ0    Y  W d   ƒ n1 sü0    Y  W | 	¡  |  	¡  n| 	¡  |  	¡  0 d S )Nr   Fr	   r  r   gš™™™™™¹?)rL   Zwait_for_workers_timeoutz7DaskDistributedBackend has no worker after 0.1 seconds.)Úmatchc                 s   s   | ]}t tƒ|ƒV  qd S r1   rM   rN   r#   r#   r)   rP   :  r+   z0test_wait_for_workers_timeout.<locals>.<genexpr>rQ   z+DaskDistributedBackend has no active workerc                 s   s   | ]}t tƒ|ƒV  qd S r1   rM   rN   r#   r#   r)   rP   @  r+   )
r   r   r   rU   rV   ÚTimeoutErrorr
   rT   ÚRuntimeErrorrŸ   )r   rD   Úmsgr#   r#   r)   Útest_wait_for_workers_timeout1  s    TV
ÿr  rL   ZlokyÚmultiprocessingc              	      s¤   t dd�}t|ƒ}zr‡ fdd„}| |¡}| ¡ }t|ƒdksBJ ‚|d j}t|tƒsZJ ‚dt|ƒv sjJ ‚W |j	dd	� |j	dd	� n|j	dd	� |j	dd	� 0 d S )
Nr	   )rk   c                     sN   t jdd��.} tdˆ d�dd„ tdƒD ƒƒ W d   ƒ n1 s@0    Y  | S )NT©Úrecordr	   )rf   rL   c                 s   s   | ]}t tƒ|ƒV  qd S r1   rM   rN   r#   r#   r)   rP   R  r+   zftest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallel.<locals>.<genexpr>rQ   )ÚwarningsÚcatch_warningsr
   rT   r  rK   r#   r)   Úfunc_using_joblib_parallelL  s    <zStest_joblib_warning_inside_dask_daemonic_worker.<locals>.func_using_joblib_parallelr   r   zdistributed.worker.daemonr™   rš   )
r   r   rÂ   rÅ   rB   Úmessagerb   ÚUserWarningrÄ   rŸ   )rL   r   rD   r  Zfutr  Úwarningr#   rK   r)   Ú/test_joblib_warning_inside_dask_daemonic_workerF  s    



ÿr  )r5   )r   r   )PÚ
__future__r   r   r   r$   r  r   r   r   Úuuidr   rU   Ú r
   r   r   r   Z_daskr   rc   r   r   Úcommonr   r   Ztest_parallelr   r   r   rœ   r   r   r   r   r   Zdistributed.metricsZdistributed.utils_testr   r   r   Zfixturer0   r4   r9   rG   r\   re   ÚmarkZparametrizerm   rp   r{   r|   r€   rŒ   r£   Úobjectr¤   r³   rÎ   rÔ   rC   rT   rÝ   rä   rê   rí   rñ   rð   ró   rö   rø   rú   rý   rÿ   ZskipifrÉ   r  r  r  r#   r#   r#   r)   Ú<module>   s‚   





7
V 
þ