a
    Žù»jþE  ã                   @   s  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZ dd	lmZ e d
d¡ZG dd„ deƒZdd„ Z G dd„ ded�Z!G dd„ de"ƒZ#G dd„ de!e#ƒZ$dS )z.Storage providers backends for Memory caching.é    N)ÚABCMetaÚabstractmethod)ÚPicklingErroré   )Únumpy_pickle)Úconcurrency_safe_rename)Úmemstr_to_bytesÚmkdirpÚ
rm_subdirs)Úformat_timeÚCacheItemInfozpath size last_accessc                   @   s   e Zd ZdZdS )ÚCacheWarningz:Warning to capture dump failures except for PicklingError.N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__© r   r   ú<E:\soccer\ml_env\Lib\site-packages\joblib/_store_backends.pyr      s   r   c                 C   s>   t t ¡ ƒ}|› dt ¡ j› dt ¡ › d|› �}|| |ƒ |S )z>Writes an object into a unique file in a concurrency-safe way.Ú.ú-)ÚidÚ	threadingÚcurrent_threadÚuuidÚuuid4ÚhexÚosÚgetpid)Zobject_to_writeÚfilenameÚ
write_funcZ	thread_idÚtemporary_filenamer   r   r   Úconcurrency_safe_write    s    $
r!   c                   @   sp   e Zd ZdZdZedd„ ƒZedd„ ƒZedd„ ƒZed	d
„ ƒZ	edd„ ƒZ
edd„ ƒZedeƒ fdd„ƒZdS )ÚStoreBackendBasez^Helper Abstract Base Class which defines all methods that
    a StorageBackend must implement.Nc                 C   s   dS )aß  Opens an item on the store and return a file-like object.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        f: a file-like object
            The file-like object where an item is stored and retrieved
        mode: string, optional
            the mode in which the file-like object is opened allowed valued are
            'rb', 'wb'

        Returns
        -------
        a file-like object
        Nr   )ÚselfÚfÚmoder   r   r   Ú
_open_item5   s    zStoreBackendBase._open_itemc                 C   s   dS )a¥  Checks if an item location exists in the store.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        location: string
            The location of an item. On a filesystem, this corresponds to the
            absolute path, including the filename, of a file.

        Returns
        -------
        True if the item exists, False otherwise
        Nr   ©r#   Úlocationr   r   r   Ú_item_existsH   s    zStoreBackendBase._item_existsc                 C   s   dS )a-  Moves an item from src to dst in the store.

        This method is private and only used by the StoreBackendMixin object.

        Parameters
        ----------
        src: string
            The source location of an item
        dst: string
            The destination location of an item
        Nr   )r#   ÚsrcÚdstr   r   r   Ú
_move_itemY   s    zStoreBackendBase._move_itemc                 C   s   dS )zÎCreates a location on the store.

        Parameters
        ----------
        location: string
            The location in the store. On a filesystem, this corresponds to a
            directory.
        Nr   r'   r   r   r   Úcreate_locationg   s    z StoreBackendBase.create_locationc                 C   s   dS )zèClears a location on the store.

        Parameters
        ----------
        location: string
            The location in the store. On a filesystem, this corresponds to a
            directory or a filename absolute path
        Nr   r'   r   r   r   Úclear_locationr   s    zStoreBackendBase.clear_locationc                 C   s   dS )z»Returns the whole list of items available in the store.

        Returns
        -------
        The list of items identified by their ids (e.g filename in a
        filesystem).
        Nr   ©r#   r   r   r   Ú	get_items}   s    zStoreBackendBase.get_itemsr   c                 C   s   dS )a—  Configures the store.

        Parameters
        ----------
        location: string
            The base location used by the store. On a filesystem, this
            corresponds to a directory.
        verbose: int
            The level of verbosity of the store
        backend_options: dict
            Contains a dictionary of named parameters used to configure the
            store backend.
        Nr   )r#   r(   ÚverboseÚbackend_optionsr   r   r   Ú	configure‡   s    zStoreBackendBase.configure)r   r   r   r   r(   r   r&   r)   r,   r-   r.   r0   Údictr3   r   r   r   r   r"   /   s    







	r"   )Ú	metaclassc                   @   s¢   e Zd ZdZd&dd„Zd'dd„Zdd	„ Zd
d„ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zd(dd„Zdd„ Zdd„ Zdd„ Zd)dd„Zd*d d!„Zd"d#„ Zd$d%„ ZdS )+ÚStoreBackendMixina‰  Class providing all logic for managing the store in a generic way.

    The StoreBackend subclass has to implement 3 methods: create_location,
    clear_location and configure. The StoreBackend also has to provide
    a private _open_item, _item_exists and _move_item methods. The _open_item
    method has to have the same signature as the builtin open and return a
    file-like object.
    r   Nc                 C   sL  t jj| jg|¢R Ž }|dkr¾|dur<d tt ¡ | ƒ¡nd}t j |d ¡}|durŠd|v rŠd dd	„ |d  ¡ D ƒ¡}|d
 |¡7 }d ||¡}	|dk r®t	d |	¡ƒ nt	d |	|¡ƒ t
| dƒsÌdn| j}
t j |d¡}|  |¡sötd| ƒ‚|
du �r:|  |d¡�}t |¡}W d  ƒ n1 �s.0    Y  ntj||
d�}|S )z:Load an item from the store given its id as a list of str.r   Nz{: <16}Ú r   Z
input_argsz, c                 s   s   | ]}d j |Ž V  qdS )z{}={}N)Úformat©Ú.0Úitemr   r   r   Ú	<genexpr>®   s   z.StoreBackendMixin.load_item.<locals>.<genexpr>z({})z[Memory]{}: Loading {}é
   z{0}...z{0} from {1}Ú	mmap_modeú
output.pklzANon-existing item (may have been cleared).
File %s does not existÚrb)r>   )r   ÚpathÚjoinr(   r8   r   ÚtimeÚbasenameÚitemsÚprintÚhasattrr>   r)   ÚKeyErrorr&   r   Úload)r#   Úcall_idr1   Ú	timestampÚmetadataÚ	full_pathZ	ts_stringÚ	signatureÚkwargsÚmsgr>   r   r$   r;   r   r   r   Ú	load_item¢   s:    ÿý

ÿ
ÿÿ
,zStoreBackendMixin.load_itemc              
      s¨   zjt jjˆ jg|¢R Ž }ˆ  |¡s,ˆ  |¡ t j |d¡}|dkrNtd| ƒ ‡ fdd„}ˆ  |||¡ W n8 ty¢ } z t	 
d|› d�t¡ W Y d}~n
d}~0 0 dS )	z;Dump an item in the store at the id given as a list of str.r?   r=   zPersisting in %sc                    s€   ˆ   |d¡�`}ztj| |ˆ jd� W n8 ty\ } z t d|› d�t¡ W Y d }~n
d }~0 0 W d   ƒ n1 sr0    Y  d S )NÚwb)ÚcompresszjUnable to cache to disk: failed to pickle output. In version 1.5 this will raise an exception. Exception: r   )r&   r   ÚdumprS   r   ÚwarningsÚwarnÚFutureWarning)Úto_writeÚdest_filenamer$   Úer/   r   r   r   Ó   s    þüz/StoreBackendMixin.dump_item.<locals>.write_funcz`Unable to cache to disk. Possibly a race condition in the creation of the directory. Exception: r   N)r   rA   rB   r(   r)   r-   rF   Ú_concurrency_safe_writeÚ	ExceptionrU   rV   r   )r#   rJ   r;   r1   Ú	item_pathr   r   rZ   r   r/   r   Ú	dump_itemÉ   s     

ÿýzStoreBackendMixin.dump_itemc                 C   s.   t jj| jg|¢R Ž }|  |¡r*|  |¡ dS )z1Clear the item at the id, given as a list of str.N©r   rA   rB   r(   r)   r.   )r#   rJ   r]   r   r   r   Ú
clear_itemè   s    
zStoreBackendMixin.clear_itemc                 C   s.   t jj| jg|¢R Ž }t j |d¡}|  |¡S )z<Check if there is an item at the id, given as a list of str.r?   )r   rA   rB   r(   r)   )r#   rJ   r]   r   r   r   r   Úcontains_itemî   s    zStoreBackendMixin.contains_itemc                 C   s   dt jj| jg|¢R Ž iS )zReturn information about item.r(   ©r   rA   rB   r(   ©r#   rJ   r   r   r   Úget_item_infoõ   s    zStoreBackendMixin.get_item_infoc                 C   s€   zjt jj| jg|¢R Ž }t j |d¡}|  |d¡�&}t | ¡  d¡¡W  d  ƒ W S 1 s^0    Y  W n   i  Y S 0 dS )z"Return actual metadata of an item.úmetadata.jsonr@   úutf-8N)	r   rA   rB   r(   r&   ÚjsonÚloadsÚreadÚdecode)r#   rJ   r]   r   r$   r   r   r   Úget_metadataù   s    8zStoreBackendMixin.get_metadatac                    s^   zLt jjˆ jg|¢R Ž }ˆ  |¡ t j |d¡}‡ fdd„}ˆ  |||¡ W n   Y n0 dS )z Store metadata of a computation.re   c                    sF   ˆ   |d¡�&}| t | ¡ d¡¡ W d   ƒ n1 s80    Y  d S )NrR   rf   )r&   Úwriterg   ÚdumpsÚencode)rX   rY   r$   r/   r   r   r   
  s    z4StoreBackendMixin.store_metadata.<locals>.write_funcN)r   rA   rB   r(   r-   r[   )r#   rJ   rL   r]   r   r   r   r/   r   Ústore_metadata  s    
z StoreBackendMixin.store_metadatac                 C   s    t jj| jg|¢R Ž }|  |¡S )z,Check cached function is available in store.)r   rA   rB   r(   Zobject_exists©r#   rJ   Ú	func_pathr   r   r   Úcontains_path  s    zStoreBackendMixin.contains_pathc                 C   s.   t jj| jg|¢R Ž }|  |¡r*|  |¡ dS )z0Clear all items with a common path in the store.Nr_   rp   r   r   r   Ú
clear_path  s    
zStoreBackendMixin.clear_pathc                 C   s€   t jj| jg|¢R Ž }|  |¡s*|  |¡ |dur|t j |d¡}|  |d¡� }| | d¡¡ W d  ƒ n1 sr0    Y  dS )ú&Store the code of the cached function.Núfunc_code.pyrR   rf   )	r   rA   rB   r(   r)   r-   r&   rl   rn   )r#   rJ   Ú	func_coderq   r   r$   r   r   r   Ústore_cached_func_code  s    

z(StoreBackendMixin.store_cached_func_codec                 C   sn   t jj| jg|¢d‘R Ž }z@|  |d¡� }| ¡  d¡W  d  ƒ W S 1 sN0    Y  W n   ‚ Y n0 dS )rt   ru   r@   rf   N)r   rA   rB   r(   r&   ri   rj   )r#   rJ   r   r$   r   r   r   Úget_cached_func_code(  s    2z&StoreBackendMixin.get_cached_func_codec                 C   s   dt jj| jg|¢R Ž iS )z?Return information related to the cached function if it exists.r(   rb   rc   r   r   r   Úget_cached_func_info1  s    z&StoreBackendMixin.get_cached_func_infoc                 C   s   |   | j¡ dS )zClear the whole store content.N)r.   r(   r/   r   r   r   Úclear5  s    zStoreBackendMixin.clearc              	   C   sX   |   |||¡}|D ]@}| jdkr.td |¡ƒ z|  |j¡ W q tyP   Y q0 qdS )zX
        Remove the store's oldest files to enforce item, byte, and age limits.
        r=   zDeleting item {0}N)Ú_get_items_to_deleter1   rF   r8   r.   rA   ÚOSError)r#   Úbytes_limitÚitems_limitÚ	age_limitÚitems_to_deleter;   r   r   r   Úenforce_store_limits9  s    
z&StoreBackendMixin.enforce_store_limitsc                 C   sB  t |tƒrt|ƒ}|  ¡ }|s"g S tdd„ |D ƒƒ}|durF|| }nd}|dur`t|ƒ| }nd}|dur¢tdd„ |D ƒƒ}| ¡ dk r’tdƒ‚t	j	 
¡ | }	nd}	|dkrÊ|dkrÊ|	du sÆ||	krÊg S |jt d¡d� g }
d}d}|D ]P}||k�r ||k�r |	du �s|	|jk �r  �q>|
 |¡ ||j7 }|d	7 }qì|
S )
zW
        Get items to delete to keep the store under size, file, & age limits.
        c                 s   s   | ]}|j V  qd S ©N)Úsizer9   r   r   r   r<   V  ó    z9StoreBackendMixin._get_items_to_delete.<locals>.<genexpr>Nr   c                 s   s   | ]}|j V  qd S r‚   )Úlast_accessr9   r   r   r   r<   c  r„   z(age_limit has to be a positive timedeltar…   )Úkeyr   )Ú
isinstanceÚstrr   r0   ÚsumÚlenÚminÚtotal_secondsÚ
ValueErrorÚdatetimeÚnowÚsortÚoperatorÚ
attrgetterr…   Úappendrƒ   )r#   r}   r~   r   rE   rƒ   Zto_delete_sizeZto_delete_itemsZ
older_itemÚdeadliner€   Zsize_so_farZitems_so_farr;   r   r   r   r{   K  sZ    

ÿþýýÿþýý


z&StoreBackendMixin._get_items_to_deletec                 C   s   t |||ƒ}|  ||¡ dS )z7Writes an object into a file in a concurrency-safe way.N)r!   r,   )r#   rX   r   r   r    r   r   r   r[   ‡  s    z)StoreBackendMixin._concurrency_safe_writec                 C   s   dj | jj| jd�S )z/Printable representation of the store location.z#{class_name}(location="{location}"))Ú
class_namer(   )r8   Ú	__class__r   r(   r/   r   r   r   Ú__repr__Œ  s    
ÿzStoreBackendMixin.__repr__)r   NN)r   )N)NN)NN)r   r   r   r   rQ   r^   r`   ra   rd   rk   ro   rr   rs   rw   rx   ry   rz   r�   r{   r[   r—   r   r   r   r   r6   ˜   s$   	
'


	

<r6   c                   @   sN   e Zd ZdZeeƒZeejj	ƒZ
eeƒZdd„ Zdd„ Zdd„ Zdd
d„Zd	S )ÚFileSystemStoreBackendz7A StoreBackend used with local or network file systems.c                 C   s&   || j krt|ƒ ntj|dd� dS )zDelete location on store.T)Úignore_errorsN)r(   r
   ÚshutilÚrmtreer'   r   r   r   r.   š  s    

z%FileSystemStoreBackend.clear_locationc                 C   s   t |ƒ dS )zCreate object location on storeN)r	   r'   r   r   r   r-   ¡  s    z&FileSystemStoreBackend.create_locationc           	         sò   g }t  | j¡D ]Ü\‰ }}t dt j ˆ ¡¡}|rt j ˆ d¡}zt j |¡}W n< t	yŒ   zt j ˆ ¡}W n t	y†   Y Y qY n0 Y n0 t
j
 |¡}z(‡ fdd„|D ƒ}tdd„ |D ƒƒ}W n t	yØ   Y qY n0 | tˆ ||ƒ¡ q|S )z7Returns the whole list of items available in the store.z[a-f0-9]{32}r?   c                    s   g | ]}t j ˆ |¡‘qS r   )r   rA   rB   ©r:   Úfn©Údirpathr   r   Ú
<listcomp>¹  r„   z4FileSystemStoreBackend.get_items.<locals>.<listcomp>c                 s   s   | ]}t j |¡V  qd S r‚   )r   rA   Úgetsizerœ   r   r   r   r<   º  r„   z3FileSystemStoreBackend.get_items.<locals>.<genexpr>)r   Úwalkr(   ÚreÚmatchrA   rD   rB   Úgetatimer|   rŽ   Úfromtimestampr‰   r“   r   )	r#   rE   Ú_Ú	filenamesZis_cache_hash_dirZoutput_filenamer…   Zfull_filenamesÚdirsizer   rž   r   r0   ¥  s(    
z FileSystemStoreBackend.get_itemsr   Nc           	   
   C   s8  |du ri }|| _ tj | j ¡s*t| j ƒ tj |¡rRtj |¡dkrRtj |¡n|}tj |d¡}tj |¡sðzBt|dƒ�$}| 	d¡ | 	d¡ W d  ƒ n1 s¦0    Y  W n< t
yî } z$t d|› d|› d	�¡ W Y d}~n
d}~0 0 | d
d¡| _| d¡}| j�r(|du�r(tjddd� || _|| _dS )zsConfigure the store backend.

        For this backend, valid store options are 'compress' and 'mmap_mode'
        NZjoblibz
.gitignoreÚwz## Created by joblib automatically.
z*
zUnable to write z. Exception: r   rS   Fr>   zSCompressed items cannot be memmapped in a filesystem store. Option will be ignored.é   )Ú
stacklevel)r(   r   rA   Úexistsr	   ÚdirnamerD   rB   Úopenrl   r|   rU   rV   ÚgetrS   r>   r1   )	r#   r(   r1   r2   Zcache_directoryZ	gitignoreÚfilerZ   r>   r   r   r   r3   Å  s8    


ÿÿý
,.
ýz FileSystemStoreBackend.configure)r   N)r   r   r   r   Ústaticmethodr¯   r&   r   rA   r­   r)   r   r,   r.   r-   r0   r3   r   r   r   r   r˜   “  s    r˜   )%r   ÚcollectionsrŽ   rg   r‘   r   Úos.pathr£   rš   r   rC   r   rU   Úabcr   r   Úpickler   r7   r   Z	backportsr   Zdiskr   r	   r
   Úloggerr   Ú
namedtupler   ÚWarningr   r!   r"   Úobjectr6   r˜   r   r   r   r   Ú<module>   s2   i |