a
    Žù»j’¤  ã                
   @   sø  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlmZ ddlmZ zddlZW n ey¦   dZY n0 ddlZddlmZmZ ddlmZmZmZmZmZ ddlmZm Z m!Z!m"Z" ddl#m$Z$ dd	l%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ dd
l,m-Z-m.Z.m/Z/ g Z0dZ1e0 2e1¡ e3Z4e0 2e4¡ e5dƒZ6e0 2e6¡ e7dƒZ8e0 2e8¡ e9dƒZ:e0 2e:¡ e;dƒZ<e0 2e<¡ e=dƒZ>e0 2e>¡ dZ?e0 2e?¡ g Z@e0 2e@¡ i ZAe0 2eA¡ eBZCe0 2eC¡ dd„ ZDG dd„ dƒZEG dd„ deFƒZGe0 2eD¡ e0 2eE¡ e0 2eG¡ eEƒ ZHe0 2eH¡ eGƒ ZIe0 2eI¡ e-dddgƒe-de0ƒdd„ ƒƒZJdd„ ZKe-dddeLƒ gƒdd„ ƒZMe*e-dg d¢ƒdd „ ƒƒZNe*d!d"„ ƒZOe*d#d$„ ƒZPe*d%d&„ ƒZQe*d'd(„ ƒZRe*d)d*„ ƒZSe*e)e-dd+d,gƒd-d.„ ƒƒƒZTe*d/d0„ ƒZUe*d1d2„ ƒZVd3d4„ ZWdªd5d6„ZXe*d7d8„ ƒZYe*d9d:„ ƒZZe*d;d<„ ƒZ[e-d=d>d?gƒd@dA„ ƒZ\e-dBdCdDdE ]dF¡fdGdH ]dF¡fgƒdIdJ„ ƒZ^e-dKdLdMgƒdNdO„ ƒZ_e*e-dg dP¢ƒe-dQeƒdRdS„ ƒƒƒZ`dTdU„ ZadVdW„ Zbe-dXdYebfdZeafgƒd[d\„ ƒZce-d]g d^¢ƒd_d`„ ƒZde*dadb„ ƒZee*dcdd„ ƒZfe*dedf„ ƒZge*dgdh„ ƒZhe*didj„ ƒZie-dkdldmdn ]e jddo¡do ¡ kdp¡ gdqdrgds�e-dtg du¢ƒdvdw„ ƒƒZle-dxdddydzdi gƒd{d|„ ƒZme-d}g d~¢ƒdd€„ ƒZne-d�ddi gƒd‚dƒ„ ƒZoe'du�rG d„d…„ d…e'jpƒZqd†d‡„ ZrG dˆd‰„ d‰ƒZse*dŠd‹„ ƒZtdŒd�„ Zue*dŽd�„ ƒZve*d�d‘„ ƒZwe*d’d“„ ƒZxe*d”d•„ ƒZyd–d—„ Zze-d˜ddi gƒd™dš„ ƒZ{d›dœ„ Z|G d�dž„ džeƒZ}G dŸd „ d eƒZ~d¡d¢„ Ze(d£d¤„ ƒZ€e+d¥d¦„ ƒZ�ej‚gZƒej„ej‚k�rÞeƒ 2ej„¡ e*e-d§eƒƒd¨d©„ ƒƒZ…dS )«z@Test the numpy pickler as a replacement of the standard pickler.é    N)Úclosing)ÚPath)Únumpy_pickleÚregister_compressor)Ú_COMPRESSORSÚ_LZ4_PREFIXÚLZ4_NOT_INSTALLED_ERRORÚBinaryZlibFileÚCompressorWrapper)Ú_IO_BUFFER_SIZEÚ_detect_compressorÚ_ensure_native_byte_orderÚ#_is_numpy_array_byte_order_mismatch)Údata)Úmemory_usedÚnpÚwith_lz4Úwith_memory_profilerÚ
with_numpyÚwithout_lz4)ÚparametrizeÚraisesÚwarnsé   © c                 c   s
   | V  d S ©Nr   )Úxr   r   úCE:\soccer\ml_env\Lib\site-packages\joblib/test/test_numpy_pickle.pyÚ	_functionU   s    r   c                   @   s   e Zd Zdd„ ZdS )Ú_classc                 C   s   d S r   r   ©Úselfr   r   r   Ú_methodZ   s    z_class._methodN©Ú__name__Ú
__module__Ú__qualname__r"   r   r   r   r   r   Y   s   r   c                   @   s   e Zd Zdd„ ZdS )Ú	_newclassc                 C   s   d S r   r   r    r   r   r   r"   _   s    z_newclass._methodNr#   r   r   r   r   r'   ^   s   r'   ÚcompressÚmemberc                 C   sD   |   d¡j}tj|||d� t |¡}|t |¡kr@||ks@J ‚d S )Nútest.pkl©r(   )ÚjoinÚstrpathr   ÚdumpÚloadÚcopyÚdeepcopy)Útmpdirr(   r)   ÚfilenameZ_memberr   r   r   Útest_standard_typesp   s
    
r4   c                   C   s:   t tƒ� t dtƒ ¡ W d   ƒ n1 s,0    Y  d S )NÚfoo)r   Ú
ValueErrorr   r.   Údictr   r   r   r   Útest_value_error}   s    
r8   Úwrong_compresséÿÿÿÿé
   c                 C   sP   d  | ¡}ttƒ� }tjdd| d� W d   ƒ n1 s80    Y  | |¡ d S )Nz%Non valid compress level given: "{0}"Údummyr5   r+   )Úformatr   r6   r   r.   Úmatch)r9   Zexception_msgÚexcinfor   r   r   Útest_compress_level_errorƒ   s    

.r@   )FTr   é   Úzlibc                 C   sÐ  |   d¡j}tj d¡}| d¡}t|f|jf||f|||gfƒD ]Š\}}tj	|||d�}t
|ƒdksjJ ‚|d |kszJ ‚tj |d ¡sŽJ ‚t |¡}|D ]}	t|	tjƒsœJ ‚qœtj t |¡t |¡¡ qBtj|d ddtjd	�}tj	|||d�}t
|ƒdk�sJ ‚t |¡}t|ƒtju�rBt|d
ƒ�rBt|t|ƒƒ�sBJ ‚tj ||¡ tƒ }tj	|||d�}t
|ƒdk�sxJ ‚t |¡}
t|
t|ƒƒ�s–J ‚tj |
j|j¡ tj |
j|j¡ tj |
j|j¡ d S )Nr*   r   )r;   é   r+   r   Úmmapúw+é   )ÚmodeÚshapeÚdtypeZ__array_prepare__)r,   r-   r   ÚrandomÚRandomStateÚrandom_sampleÚ	enumerateÚTr   r.   ÚlenÚosÚpathÚexistsr/   Ú
isinstanceÚndarrayÚtestingÚassert_array_equalÚarrayÚmemmapÚfloat64ÚtypeÚhasattrÚComplexTestObjectÚarray_floatÚ	array_intÚ	array_obj)r2   r(   r3   ÚrndÚaÚindexÚobjÚ	filenamesZobj_ÚitemÚ
obj_loadedr   r   r   Útest_numpy_persistenceŒ   s6    
(


rg   c                 C   sL   t jtd t jd�}|  d¡j}tj||dd� t |¡}t j	 
||¡ d S )Néd   ©rI   r*   Tr+   )r   Úonesr   Úuint8r,   r-   r   r.   r/   rU   rV   )r2   Ú	big_arrayr3   Zarr_reloadedr   r   r   Ú2test_numpy_persistence_bufferred_array_compression¿   s
    
rm   c                 C   sä  t j d¡}| d¡}|  d¡j}t ||¡ tj|dd�}t	|t j
ƒsLJ ‚|  d¡j}tƒ }t ||¡ tj|dd�}t	|t|ƒƒsŠJ ‚t	|jt j
ƒsœJ ‚|jjjrªJ ‚t	|jt j
ƒs¼J ‚|jjjrÊJ ‚t	|jt j
ƒrÜJ ‚t j |j|j¡ t j |j|j¡ t j |j|j¡ tj|dd�}|jjj�s0J ‚d|jdd…< |jjj�sNJ ‚d|jdd…< tj|dd�}t j |j|j¡ t j |j|j¡ tj|d	d� |jjj�s¬J ‚|jjdk�s¾J ‚|jjj�sÎJ ‚|jjdk�sàJ ‚d S )
Nr   r;   z	test1.pklÚr©Ú	mmap_modez	test2.pklúr+g      $@rE   )r   rJ   rK   rL   r,   r-   r   r.   r/   rS   rX   r\   rZ   r]   ÚflagsZ	writeabler^   r_   rU   rV   rG   )r2   r`   ra   r3   Úbrc   rf   Úobj_reloadedr   r   r   Útest_memmap_persistenceÉ   s@    
ru   c                 C   s~   t j d¡}| d¡}t jddgtd�}||f}|  d¡j}t 	||¡ tj
|dd�\}}t|t jƒsjJ ‚t|t jƒrzJ ‚d S )	Nr   r;   r   rs   ri   r*   rn   ro   )r   rJ   rK   rL   rW   Úobjectr,   r-   r   r.   r/   rS   rX   )r2   r`   ra   rs   Z	constructr3   Za_cloneZb_cloner   r   r   Ú$test_memmap_persistence_mixed_dtypesö   s    
rw   c                 C   s`   t j d¡}| d¡}t j |d¡}|  d¡j}t 	||¡ tj
|dd�}t|t jjƒs\J ‚d S )Nr   r;   g      à?r*   rn   ro   )r   rJ   rK   rL   ÚmaZmasked_greaterr,   r-   r   r.   r/   rS   Zmasked_array)r2   r`   ra   r3   rs   r   r   r   Útest_masked_array_persistence	  s    
ry   c                 C   sÜ   t j d¡}| d¡}|  d¡j}tj||dd� tt	ƒ�}tj
|dd�}W d   ƒ n1 s^0    Y  d d	d
„ |D ƒ¡}dd
„ |D ƒ}t|t jƒršJ ‚t j ||¡ t|ƒdks¼J |ƒ‚t|d ƒd|› d�ksØJ ‚d S )Nr   r;   r*   r   r+   rq   ro   Ú
c                 S   s   g | ]}t |ƒ‘qS r   )Ústr©Ú.0Úwr   r   r   Ú
<listcomp>  ó    z3test_compress_mmap_mode_warning.<locals>.<listcomp>c                 S   s   g | ]
}|j ‘qS r   ©Úmessager|   r   r   r   r      r€   z6mmap_mode "r+" is not compatible with compressed file z. "r+" flag will be ignored.)r   rJ   rK   rL   r,   r-   r   r.   r   ÚUserWarningr/   rS   rX   rU   rV   rO   r{   )r2   r`   rc   Zthis_filenameÚwarninfoZreloaded_objZ	debug_msgr   r   r   Útest_compress_mmap_mode_warning  s    

,ÿÿr…   TFc                 C   sª   |   d¡j}t d¡}tjdtdƒ tjd�}||fD ]n}|jd }|ttj 	dd¡ƒ }t
tj|||d�}td	 }	||	ks€J ‚t
tj|ƒ}d
t }
|||
 k s6J ‚q6d S )Nr*   ©r;   r;   rh   g    €„.A©rH   rI   r   éè  r+   g�íµ ÷Æ0@é    )r,   r-   r   rj   Úintrk   Únbytesr{   rJ   Úrandintr   r   r.   r   r/   )r2   r(   r3   Zsmall_arrayrl   rc   ÚsizeZobj_filenameZmem_usedZwrite_buf_sizeZread_buf_sizer   r   r   Útest_memory_usage*  s    

rŽ   c              
   C   s  t jdt  d¡d�t jdt  d¡d�t jdt  d¡d�t jdt  d¡d�t jdddd	d
œgdd�t jdt jd� ¡ dg}|  d¡j}tj	||dd�}t
|ƒdks¤J ‚t |¡}t||ƒD ]H\}}t|t jƒrôt|ƒ}|j|jksäJ ‚t j ||¡ q¸||ks¸J ‚q¸d S )Né   ú<i8ri   ú>i8ú<f8ú>f8r   ÚabcrC   ©ra   rs   ÚOé   õ   C'est l'Ã©tÃ© !ztemp.pkl.gzr+   )r   ÚarangerI   rW   rk   Útobytesr,   r-   r   r.   rO   r/   ÚziprS   rT   r   rU   Úassert_equal)r2   Úexpected_listÚfnameZdumped_filenamesÚresult_listÚresultÚexpectedr   r   r   Ú$test_compressed_pickle_dump_and_loadD  s$    ù

r¢   c                 C   sº   t  d¡}t  d¡}||f}t jd|d�}t jd|d�}|  d¡j}t ||g|¡ tj|dd�\}}|j|jksvJ ‚|j|v s„J ‚tj|dd�\}	}
|	j|jks¦J ‚|
j|jks¶J ‚d S )	Nr�   r‘   r�   ri   ztemp.pklT)Úensure_native_byte_orderF)r   rI   r™   r,   r-   r   r.   r/   )r2   Zlittle_endian_dtypeZbig_endian_dtypeZ
all_dtypesZle_arrayZbe_arrayrž   Zle_array_native_loadZbe_array_native_loadZle_array_nonnative_loadZbe_array_nonnative_loadr   r   r   Útest_memmap_load^  s"    

ÿ
ÿ
r¤   c                  C   sJ   d} t tt | ¡d��  tjdddd� W d   ƒ n1 s<0    Y  d S )NzrNative byte ordering can only be enforced if 'mmap_mode' parameter is set to None, but got 'mmap_mode=r+' instead.)r>   z/path/to/some/dump.pklTrq   )r£   rp   )r   r6   ÚreÚescaper   r/   )Zexpected_msgr   r   r   Útest_invalid_parameters_raisez  s    ÿÿr§   c              
   C   sÒ  t  d| ¡}t| d¡ƒ}dddœ}| dd¡}| |d¡}||k�rr�z�tjdd��(}t d	¡ tj	| |d
�}	W d  ƒ n1 s„0    Y  t
j | ¡}
d|
v sªd|
v r®dnd}t  d|
¡rÊ|durÊdnd}|| }t|ƒ|k�sJ d|› ddd„ |D ƒ› �ƒ‚dd„ |D ƒ}dd„ |D ƒ}|D ] }t|jƒd | ¡k�s"J ‚�q"|D ]0}t  | ¡}t  d|› d�t|jƒ¡�sHJ ‚�qHt|	|ƒD ]P\}}t|tjƒ�rÄt|ƒ}|j|jk�s´J ‚tj ||¡ n||k�s„J ‚�q„W n– t�yn } z||dk�rt|tƒ�sJ ‚d}|t|ƒv �sZJ ‚n>|  d¡�rXtjd �rXt|tƒ�sDJ ‚t t|ƒv �sZJ ‚n‚ W Y d}~n
d}~0 0 n\zt 	| ¡ t!dƒ‚W nD t�yÌ } z*d |¡}|t|jƒv �s¸J ‚W Y d}~n
d}~0 0 dS )z�Helper function to test joblib pickle content.

    Note: currently only pickles containing an iterable are supported
    by this function.
    z.+py(\d)(\d).+r   rC   rA   )rC   rA   rF   T)ÚrecordÚalwaysro   Nz_0.9z_0.8.4r   z_0.1.+.pkl$z6Did not get the expected number of warnings. Expected z but got warnings: c                 S   s   g | ]
}|j ‘qS r   r�   r|   r   r   r   r   ©  r€   z!_check_pickle.<locals>.<listcomp>c                 S   s   g | ]}t |jtƒr|‘qS r   )Ú
issubclassÚcategoryÚDeprecationWarningr|   r   r   r   r   ¬  s   c                 S   s   g | ]}t |jtƒr|‘qS r   )rª   r«   rƒ   r|   r   r   r   r   ¯  r€   zkThe file '{0}' has been generated with a joblib version less than 0.10. Please regenerate this pickle file.zmemmapped.+z.+segmentation faultzPYou may be trying to read with python 3 a joblib pickle generated with python 2.ú.lz4z>Numpy pickle loading should have raised a ValueError exceptionz unsupported pickle protocol: {0})"r¥   r>   rŠ   ÚgroupÚgetÚwarningsÚcatch_warningsÚsimplefilterr   r/   rP   rQ   ÚbasenameÚsearchrO   r{   r‚   r=   r¦   r›   rS   r   rT   r   rI   rU   rœ   Ú	Exceptionr6   Úendswithr   Úargsr   ÚAssertionError)r3   r�   rp   Zversion_matchZpy_version_used_for_writingZ%py_version_to_default_pickle_protocolZpickle_reading_protocolZpickle_writing_protocolr„   rŸ   Úfilename_baseZ expected_nb_deprecation_warningsZexpected_nb_user_warningsZexpected_nb_warningsZdeprecation_warningsZuser_warningsr~   Zescaped_filenamer    r¡   Úexcr‚   Úer   r   r   Ú_check_pickle†  sŒ    
ÿ

,ÿ
ÿÿýÿÿþÿÿþÿÿ
ÿ
ÿ
ÿ
r¼   c               	      sÆ   t jdt  d¡d�t jdt  d¡d�t jdddddœgd	d�t jd
t jd� ¡ t jg d¢t  d¡d�dg} tj 	tj 
tj¡¡‰d‰ td ur”ˆ d7 ‰ ‡ ‡fdd„t ˆ¡D ƒ}|D ]}t|| ƒ q²d S )Nr�   r�   ri   r’   r   r”   rC   r•   r–   r—   ©r   r   rC   r˜   )ú.pklú.gzz.gzipú.bz2Úlz4)ú.xzú.lzmac                    s0   g | ](‰ t ‡ fd d„ˆD ƒƒrtj ˆˆ ¡‘qS )c                 3   s   | ]}ˆ   |¡V  qd S r   )r¶   )r}   Úext©Úfnr   r   Ú	<genexpr>ÿ  r€   zGtest_joblib_pickle_across_python_versions.<locals>.<listcomp>.<genexpr>)ÚanyrP   rQ   r,   )r}   ©Zpickle_extensionsÚtest_data_dirrÅ   r   r   ü  s   þz=test_joblib_pickle_across_python_versions.<locals>.<listcomp>)r   r™   rI   rW   rk   rš   ÚmatrixrP   rQ   ÚdirnameÚabspathr   Ú__file__ÚlzmaÚlistdirr¼   ©r�   Zpickle_filenamesrž   r   rÉ   r   Ú)test_joblib_pickle_across_python_versionsá  s     ÷þrÒ   c               	      s´   t jdt  d¡d�t jdt  d¡d�t jdddddœgd	d�t jd
t jd� ¡ t jg d¢t  d¡d�dg} tj 	tj 
tj¡¡‰ ‡ fdd„t ˆ ¡D ƒ}|D ]}t|| dd� qœd S )Nr�   r�   ri   r’   r   r”   rC   r•   r–   r—   r½   r˜   c                    s$   g | ]}|  d ¡rtj ˆ |¡‘qS )r¾   )r¶   rP   rQ   r,   )r}   rÆ   ©rÊ   r   r   r     s   
þzGtest_joblib_pickle_across_python_versions_with_mmap.<locals>.<listcomp>rn   ro   )r   r™   rI   rW   rk   rš   rË   rP   rQ   rÌ   rÍ   r   rÎ   rÐ   r¼   rÑ   r   rÓ   r   Ú3test_joblib_pickle_across_python_versions_with_mmap  s    ÷
þrÔ   c                  C   sf  t jddgddgd�t jdt  d¡d�t jdt  d¡d�g} | D ]j}tjd	kr\t|ƒrhJ ‚nt|ƒshJ ‚t|ƒ}|jjrš|jj 	¡ D ]}|d
 jdk q„q@|jjdks@J ‚q@t jddgddgd�t jdt  d¡d�t jdt  d¡d�g}|D ]t}tjdk�rt|ƒ�rJ ‚nt|ƒ�sJ ‚t|ƒ}|jj�rP|jj 	¡ D ]}|d
 jdk �q8qì|jjdksìJ ‚qìd S )N)r   g       @)rA   g      @)Ú r‘   )rÕ   r“   ri   rA   r‘   r“   Úbigr   ú=)rÕ   r�   )rÕ   r’   r�   r’   Úlittle)
r   rW   r™   rI   ÚsysÚ	byteorderr   r   ÚfieldsÚvalues)Z	be_arraysrW   Z	convertedÚfZ	le_arraysr   r   r   Ú.test_numpy_array_byte_order_mismatch_detection  s4    ý
ý
rÞ   Úcompress_tuple)rB   rA   ©ÚgziprA   c                 C   s^   |   d¡j}tjd||d� t|dƒ�$}t|ƒ|d ks<J ‚W d   ƒ n1 sP0    Y  d S )Nr*   r<   r+   Úrbr   ©r,   r-   r   r.   Úopenr   )r2   rß   r3   rÝ   r   r   r   Útest_compress_tuple_argumentJ  s    rå   zcompress_tuple,message))rB   rA   Úextraz9Compress argument tuple should contain exactly 2 elements)ÚwrongrA   z(Non valid compression method given: "{}"rç   )rB   rç   z$Non valid compress level given: "{}"c                 C   sR   |   d¡j}ttƒ� }tjd||d� W d   ƒ n1 s:0    Y  | |¡ d S )Nr*   r<   r+   )r,   r-   r   r6   r   r.   r>   )r2   rß   r‚   r3   r?   r   r   r   Ú&test_compress_tuple_argument_exceptionT  s    
.rè   Úcompress_stringrB   rá   c                 C   sZ   |   d¡j}tjd||d� t|dƒ� }t|ƒ|ks8J ‚W d   ƒ n1 sL0    Y  d S )Nr*   r<   r+   râ   rã   )r2   ré   r3   rÝ   r   r   r   Útest_compress_string_argumentm  s    rê   )r   rA   é   Úcmethodc           	   	   C   s  |   d¡j}tjddd�tdƒdddœg d	i d
df}|dv rPtd u rPt d¡ n|dkrltj	d
 rlt d¡ |d | }|D ]˜}t
j||||fd� t|dƒ� }t|ƒ|ks°J ‚W d   ƒ n1 sÄ0    Y  t
 |¡}t|t|ƒƒsêJ ‚t|tjƒ�rtj ||¡ q|||ks|J ‚q|d S )Nr*   )rh   rh   Zf8r‡   r;   r   rs   )ra   rC   r   r   g      ð?©rÏ   Úxzzlzma is support not availablerÁ   zlz4 is not installed.Ú.r+   râ   )r,   r-   r   rj   ÚrangerÏ   ÚpytestÚskipr   r·   r   r.   rä   r   r/   rS   rZ   rT   rU   rV   )	r2   r(   rì   r3   ÚobjectsZdump_filenamerc   rÝ   rt   r   r   r   Útest_joblib_compression_formatsw  s0    ø
.
rô   c                 C   sp   t t | d¡ƒ�}| ¡ }W d  ƒ n1 s.0    Y  t|dƒ�}| |¡ W d  ƒ n1 sb0    Y  dS )zDecompress a gzip file.râ   NÚwb)r   rá   ÚGzipFileÚreadrä   Úwrite©Zsource_filenameZtarget_filenameÚfoÚbufr   r   r   Ú_gzip_file_decompressž  s    &rü   c                 C   sp   t | dƒ�}t | ¡ ¡}W d  ƒ n1 s.0    Y  t |dƒ�}| |¡ W d  ƒ n1 sb0    Y  dS )zDecompress a zlib file.râ   Nrõ   )rä   rB   Ú
decompressr÷   rø   rù   r   r   r   Ú_zlib_file_decompress§  s    ,rþ   zextension,decompressú.zr¿   c                 C   sH   d}|   d¡j}|| }t ||¡ |||ƒ t |¡}||ksDJ ‚d S )Nza string to persistr*   )r,   r-   r   r.   r/   )r2   Ú	extensionrý   rc   Zfilename_rawZfilename_compressedrt   r   r   r   Ú'test_load_externally_decompressed_files°  s    

r  zextension,cmethod))rÿ   rB   )r¿   rá   )rÀ   Úbz2)rÃ   rÏ   )rÂ   rî   )r¾   únot-compressed)rÕ   r  c                 C   s¤   |dv rt d u rt d¡ |  d¡j}d}|| }t ||¡ t|dƒ� }t|ƒ|ksZJ ‚W d   ƒ n1 sn0    Y  t 	|¡}t
|t|ƒƒs”J ‚||ks J ‚d S )Nrí   zlzma is missingr*   zobject to dumprâ   )rÏ   rñ   rò   r,   r-   r   r.   rä   r   r/   rS   rZ   )r2   r   rì   r3   rc   Z
dump_fnamerÝ   rt   r   r   r   Ú%test_compression_using_file_extensionÆ  s    
.
r  c           	   
   C   s8  t j d¡dg}tjtjg}td ur0|tjg7 }|  d¡j	}|D ]ò}|D ]è}||dƒ�}t
 ||¡ W d   ƒ n1 sx0    Y  ||dƒ�}t
 |¡}W d   ƒ n1 s¬0    Y  t|dƒ�}t
 |¡}W d   ƒ n1 sà0    Y  t|t jƒ�rt j ||¡ t j ||¡ qH||k�s$J ‚||ksHJ ‚qHq@d S )Nr†   ú	some datar*   rõ   râ   )r   rJ   r  ÚBZ2Filerá   rö   rÏ   ÚLZMAFiler,   r-   r   r.   r/   rä   rS   rT   rU   rV   )	r2   ÚobjsZfobjsr3   rc   ÚfobjrÝ   rt   Zobj_reloaded_2r   r   r   Útest_file_handle_persistenceæ  s$    *((r
  c                  C   sd   t j d¡dg} | D ]J}t ¡ }t ||¡ t |¡}t|t jƒrRt j	 
||¡ q||ksJ ‚qd S )Nr†   r  )r   rJ   ÚioÚBytesIOr   r.   r/   rS   rT   rU   rV   )r  rc   rÝ   rt   r   r   r   Útest_in_memory_persistence  s    
r  c                 C   s˜   t j d¡}|  d¡j}t|dƒ�}t ||¡ W d   ƒ n1 sD0    Y  t|dƒ�}tj|dd�}W d   ƒ n1 s|0    Y  t j 	||¡ d S )Nr†   r*   rõ   râ   rq   ro   )
r   rJ   r,   r-   rä   r   r.   r/   rU   rV   )r2   rc   r3   rÝ   rt   r   r   r   Ú!test_file_handle_persistence_mmap  s    *,r  c              	   C   sì   t j d¡}|  d¡j}t|dƒ� }tj||dd� W d   ƒ n1 sH0    Y  tt 	|d¡ƒ�v}t
tƒ�}tj|dd� W d   ƒ n1 s�0    Y  t|ƒd	ksªJ ‚t|d
 jƒd|ddœ ksÊJ ‚W d   ƒ n1 sÞ0    Y  d S )Nr†   r*   rõ   rà   r+   râ   rq   ro   r   r   zP"%(fileobj)r" is not a raw file, mmap_mode "%(mmap_mode)s" flag will be ignored.)Úfileobjrp   )r   rJ   r,   r-   rä   r   r.   r   rá   rö   r   rƒ   r/   rO   r{   r‚   )r2   rc   r3   rÝ   r„   r   r   r   Ú,test_file_handle_persistence_compressed_mmap   s    .
,ÿÿÿr  c                  C   sˆ   t j d¡} t ¡ }t | |¡ ttƒ�}tj|dd� W d   ƒ n1 sL0    Y  t	|ƒdksfJ ‚t
|d jƒdddi ks„J ‚d S )Nr†   rq   ro   r   r   zuIn memory persistence is not compatible with mmap_mode "%(mmap_mode)s" flag passed. mmap_mode option will be ignored.rp   )r   rJ   r  r  r   r.   r   rƒ   r/   rO   r{   r‚   )rc   rû   r„   r   r   r   Ú+test_file_handle_persistence_in_memory_mmap3  s    
,þÿÿr  r   s   a little data as bytes.i'  z{}rˆ   zlatin-1za little data as bytes.za large data as bytes.)ZidsÚcompress_level)r   rA   é	   c              
   C   s   |   d¡j}t|dƒ��}t|d|d��ž}| ¡ s6J ‚| |¡ | ¡ | ¡ ksTJ ‚ttj	ƒ� | 
¡  W d   ƒ n1 s|0    Y  ttj	ƒ� | ¡  W d   ƒ n1 s®0    Y  W d   ƒ n1 sÌ0    Y  |jsàJ ‚ttƒ� | ¡  W d   ƒ n1 �s0    Y  W d   ƒ n1 �s(0    Y  t|dƒ�è}t|ƒ�²}| ¡ �sVJ ‚| ¡ �sdJ ‚| ¡ | ¡ k�szJ ‚| ¡ |k�sŒJ ‚ttj	ƒ� | ¡  W d   ƒ n1 �s¶0    Y  | ¡ �sÎJ ‚| d¡ | ¡ dk�sêJ ‚W d   ƒ n1 �s 0    Y  |j�sJ ‚W d   ƒ n1 �s,0    Y  t|d|d��(}| ¡ �sTJ ‚| |¡ W d   ƒ n1 �st0    Y  t|dƒ�0}| ¡ |k�sœJ ‚| ¡ �sªJ ‚W d   ƒ n1 �sÀ0    Y  t|d|d�}| ¡ �sæJ ‚| |¡ | ¡  t|dƒ}| ¡ |k�sJ ‚| ¡  d S )Nr*   rõ   ©Úcompresslevelrâ   r   )r,   r-   rä   r	   Úwritablerø   Úfilenor   r  ÚUnsupportedOperationÚ_check_can_readÚ_check_can_seekÚclosedr6   Ú_check_not_closedÚreadableÚseekabler÷   Ú_check_can_writeÚseekÚtellÚclose)r2   r   r  r3   rÝ   Zfzr   r   r   Útest_binary_zlibfileE  sL    
&D

H
(
2,*.

r#  Ú	bad_valueé   ra   c                 C   s`   |   d¡j}ttƒ�}t|d|d� W d   ƒ n1 s80    Y  t d |¡¡}| |¡ d S )Nr*   rõ   r  zS'compresslevel' must be an integer between 1 and 9. You provided 'compresslevel={}')	r,   r-   r   r6   r	   r¥   r¦   r=   r>   )r2   r$  r3   r?   Úpatternr   r   r   Ú+test_binary_zlibfile_bad_compression_levels�  s    
,ÿÿr'  Úbad_mode)ra   r   rn   r~   r   rC   c                 C   sL   |   d¡j}ttƒ�}t||ƒ W d   ƒ n1 s40    Y  | d¡ d S )Nr*   zInvalid mode)r,   r-   r   r6   r	   r>   )r2   r(  r3   r?   r   r   r   Ú"test_binary_zlibfile_invalid_modes�  s    
(r)  Úbad_filec                 C   s@   t tƒ�}t| dƒ W d   ƒ n1 s(0    Y  | d¡ d S )Nrâ   z1filename must be a str or bytes object, or a file)r   Ú	TypeErrorr	   r>   )r*  r?   r   r   r   Ú*test_binary_zlibfile_invalid_filename_type•  s    
(r,  c                   @   s   e Zd Zdd„ ZdS )ÚSubArrayc                 C   s   t t | ¡ffS r   )Ú_load_sub_arrayr   Zasarrayr    r   r   r   Ú
__reduce__¡  s    zSubArray.__reduce__N)r$   r%   r&   r/  r   r   r   r   r-     s   r-  c                 C   s   t | jƒ}| |d d …< |S r   )r-  rH   )ZarrÚdr   r   r   r.  ¤  s    
r.  c                   @   s   e Zd ZdZdd„ ZdS )r\   z7A complex object containing numpy arrays as attributes.c                 C   s8   t jddd�| _t jddd�| _t jg d¢dd�| _d S )Nrh   rY   ri   Zint32)ra   r;   g      4@rv   )r   r™   r]   rj   r^   rW   r_   r    r   r   r   Ú__init__¬  s    zComplexTestObject.__init__N)r$   r%   r&   Ú__doc__r1  r   r   r   r   r\   ©  s   r\   c                 C   sJ   |   d¡j}tdƒ}t ||¡ t |¡}t|tƒs8J ‚tj 	||¡ d S )Nr*   )r;   )
r,   r-   r-  r   r.   r/   rS   r   rU   rV   )r2   r3   ra   Úcr   r   r   Útest_numpy_subclass²  s    
r4  c                 C   sX   |   d¡j}d}t |t|ƒ¡ t |¡|ks2J ‚t ||¡ t t|ƒ¡|ksTJ ‚d S )Nr*   é{   )r,   r-   r   r.   r   r/   )r2   r3   Úvaluer   r   r   Útest_pathlib¼  s    r7  c                 C   s”   |   d¡j}t ddgddgg¡dd … tjddd�d d …d d…d d …f fD ]@}|jjr^J ‚|jjrjJ ‚t 	||¡ t 
|¡}tj ||¡ qNd S )	Nr*   r   rC   rA   rF   )r;   é2   é   ÚF)Úorder)r,   r-   r   Zasfortranarrayrj   rr   Úc_contiguousÚf_contiguousr   r.   r/   rU   rV   )r2   r3   rW   Úarray_reloadedr   r   r   Ú"test_non_contiguous_array_picklingÅ  s    "û
r?  c                 C   sD   |   d¡j}t d¡}tj||tjd� t |¡}tj	 
||¡ d S )Nr*   r;   ©Úprotocol)r,   r-   r   Úzerosr   r.   ÚpickleÚHIGHEST_PROTOCOLr/   rU   rV   )r2   r3   Ú
test_arrayr>  r   r   r   Útest_pickle_highest_protocol×  s
    

rF  c            
   	   C   sL  t  d¡} d}t tjtj¡}| |¡ | d¡ t |¡�ú}| ¡ \}}| 	d¡�}t
 | |¡ W d   ƒ n1 sv0    Y  | 	d¡�}t
 |¡}W d   ƒ n1 sª0    Y  t j || ¡ t ¡ }	t
 | |	¡ | |	 ¡ ¡ | 	d¡�}t
 |¡}W d   ƒ n1 �s0    Y  t j || ¡ W d   ƒ n1 �s>0    Y  d S )Nr;   )Ú	localhosti90  r   rõ   râ   )r   r™   ÚsocketÚAF_INETÚSOCK_STREAMÚbindÚlistenÚcreate_connectionÚacceptÚmakefiler   r.   r/   rU   rV   r  r  ÚsendÚgetvalue)
rE  Z_ADDRZlistenerÚclientÚserverÚclient_addrZsfÚcfr>  Zbytes_to_sendr   r   r   Útest_pickle_in_socketæ  s$    


*(*rV  c                 C   s€   |   d¡j}tj}tj|dd�tj|dd�g}t ||¡ tj	|dd�}t
|d tjƒs\J ‚|d j|ksnJ ‚tj ||¡ d S )Nú	test.mmaprk   ri   rn   ro   r   )r,   r-   rD   ÚALLOCATIONGRANULARITYr   rB  rj   r   r.   r/   rS   rX   ÚoffsetrU   rV   )r2   rž   r�   rc   Zmemmapsr   r   r   Ú test_load_memmap_with_big_offset  s    rZ  c                    sn   d}d‰G dd„ dt jƒ‰ G ‡ ‡fdd„dtƒ}t||ƒ ƒ t| jˆ ksNJ ‚t| jˆks`J ‚t |¡ d S )Nú	test-nameztest-prefixc                   @   s   e Zd ZdS )z:test_register_compressor.<locals>.BinaryCompressorTestFileN©r$   r%   r&   r   r   r   r   ÚBinaryCompressorTestFile  s   r]  c                       s   e Zd Z‡ ‡fdd„ZdS )z=test_register_compressor.<locals>.BinaryCompressorTestWrapperc                    s   t j| ˆ ˆd� d S )N©rc   Úprefix©r
   r1  r    ©r]  Zcompressor_prefixr   r   r1    s    ÿzFtest_register_compressor.<locals>.BinaryCompressorTestWrapper.__init__N©r$   r%   r&   r1  r   ra  r   r   ÚBinaryCompressorTestWrapper  s   rc  )r  ÚBufferedIOBaser
   r   r   Úfileobj_factoryr_  Úpop)r2   Úcompressor_namerc  r   ra  r   Útest_register_compressor  s    rh  Úinvalid_namec                 C   s@   t tƒ�}t| d ƒ W d   ƒ n1 s(0    Y  | d¡ d S )Nz"Compressor name should be a string)r   r6   r   r>   )ri  r?   r   r   r   Ú%test_register_compressor_invalid_name.  s    
(rj  c                     sd   G dd„ dƒ‰ G ‡ fdd„dt ƒ} ttƒ�}td| ƒ ƒ W d   ƒ n1 sL0    Y  | d¡ d S )Nc                   @   s   e Zd ZdS )zCtest_register_compressor_invalid_fileobj.<locals>.InvalidFileObjectNr\  r   r   r   r   ÚInvalidFileObject9  s   rk  c                       s   e Zd Z‡ fdd„ZdS )zJtest_register_compressor_invalid_fileobj.<locals>.InvalidFileObjectWrapperc                    s   t j| ˆ dd� d S ©Ns   prefixr^  r`  r    ©rk  r   r   r1  =  s    zStest_register_compressor_invalid_fileobj.<locals>.InvalidFileObjectWrapper.__init__Nrb  r   rm  r   r   ÚInvalidFileObjectWrapper<  s   rn  ÚinvalidzQCompressor 'fileobj_factory' attribute should implement the file object interface)r
   r   r6   r   r>   )rn  r?   r   rm  r   Ú(test_register_compressor_invalid_fileobj6  s    
*ÿrp  c                   @   s   e Zd Zdd„ ZdS )ÚAnotherZlibCompressorWrapperc                 C   s   t j| tdd� d S rl  )r
   r1  r	   r    r   r   r   r1  J  s    z%AnotherZlibCompressorWrapper.__init__Nrb  r   r   r   r   rq  I  s   rq  c                   @   s   e Zd Zdd„ ZdS )Ú StandardLibGzipCompressorWrapperc                 C   s   t j| tjdd� d S rl  )r
   r1  rá   rö   r    r   r   r   r1  O  s    z)StandardLibGzipCompressorWrapper.__init__Nrb  r   r   r   r   rr  N  s   rr  c                  C   s’   d} t | tƒ ƒ ttƒ�}t | tƒ ƒ W d   ƒ n1 s:0    Y  | d | ¡¡ t | tƒ dd� | tv spJ ‚t|  jt	j
ks„J ‚t | ¡ d S )Nr[  z#Compressor '{}' already registered.T)Úforce)r   rq  r   r6   rr  r>   r=   r   re  rá   rö   rf  )rg  r?   r   r   r   Ú+test_register_compressor_already_registeredS  s    
*rt  c                 C   s  dd l }d}|tv sJ ‚t| j|jjks.J ‚|  d¡j}d}tj|||d� t	|dƒ�&}| 
ttƒ¡tkspJ ‚W d   ƒ n1 s„0    Y  t |¡|ks J ‚t ||d ¡ t	|dƒ�&}| 
ttƒ¡tksÒJ ‚W d   ƒ n1 sæ0    Y  t |¡|k�sJ ‚d S )Nr   rÁ   r*   ú	test datar+   râ   r­   )Z	lz4.framer   re  ÚframeZLZ4FrameFiler,   r-   r   r.   rä   r÷   rO   r   r/   )r2   rÁ   Z
compressorrž   r   rÝ   r   r   r   Útest_lz4_compressionh  s    44rw  c                 C   sœ   |   d¡j}d}t}ttƒ� }tj||dd� W d   ƒ n1 sB0    Y  | |¡ ttƒ� }t ||d ¡ W d   ƒ n1 s„0    Y  | |¡ d S )Nz
test.nolz4ru  rÁ   r+   r­   )r,   r-   r   r   r6   r   r.   r>   )r2   rž   r   Úmsgr?   r   r   r   Ú test_lz4_compression_without_lz4€  s    
.

.ry  rA  c                 C   s4  |   d¡j}tj d¡}tj|||d� tj|dd�}t|tj	ƒsFJ ‚tj
 ||¡ |jjtj dksjJ ‚|jjsvJ ‚tj d¡tj d¡tj d¡tj d¡g}|   d¡j}tj|||d� tj|dd�}t|ƒD ]N\}}t|tj	ƒsìJ ‚tj
 || |¡ |jjtj dk�sJ ‚|jjsÔJ ‚qÔtjdtjd�tjd	tjd�tjd
tjd�tjdtjd�tjdtjd�tjdtjd�tjdtjd�tjdtjd�tjdtjd�dœ	}|   d¡j}tj|||d� tj|dd�}	|	 ¡ D ]T\}
}t|tj	ƒ�sôJ ‚tj
 ||
 |¡ |jjtj dk�sJ ‚|jj�sÚJ ‚�qÚd S )NrW  rC   r@  rn   ro   r   z
test1.mmapri   rA   r�   é   é   é   é   é   é   )	Za0Za1Za2Úa3Za4Za5Za6Za7Za8z
test2.mmap)r,   r-   r   rJ   Zrandnr   r.   r/   rS   rX   rU   rV   Úctypesr   ZNUMPY_ARRAY_ALIGNMENT_BYTESrr   ZalignedrM   r™   rk   Úitems)r2   rA  rž   ra   rX   Z
array_listZ
l_reloadedÚidxZ
array_dictZ
d_reloadedÚkeyr   r   r   Útest_memmap_alignment_padding”  sN    



ü÷r…  )N)†r2  r  r0   rá   r  rD   rP   rC  rJ   r¥   rH  rÙ   r°   rB   Ú
contextlibr   Úpathlibr   rÏ   ÚImportErrorrñ   Zjoblibr   r   Zjoblib.compressorr   r   r   r	   r
   Zjoblib.numpy_pickle_utilsr   r   r   r   Zjoblib.testr   Zjoblib.test.commonr   r   r   r   r   r   Zjoblib.testingr   r   r   ZtypelistZ_noneÚappendrZ   Ú_typeÚboolÚ_boolrŠ   Ú_intÚfloatZ_floatÚcomplexZ_complexr{   Ú_stringÚ_tupleZ_listÚ_dictrO   Z_builtinr   r   rv   r'   Z	_instanceÚ_objectr4   r8   r7   r@   rg   rm   ru   rw   ry   r…   rŽ   r¢   r¤   r§   r¼   rÒ   rÔ   rÞ   rå   r=   rè   rê   rô   rü   rþ   r  r  r
  r  r  r  r  rŒ   Úencoder#  r'  r)  r,  rT   r-  r.  r\   r4  r7  r?  rF  rV  rZ  rh  rj  rp  rq  rr  rt  rw  ry  ÚDEFAULT_PROTOCOLZ	protocolsrD  r…  r   r   r   r   Ú<module>   sn  
 
















1
	
,





[
$

*
	þþ÷þ

	$		þ
ý





ýù	2



	
		






