a j1@s dZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZddlmZddlmZddlmZddlZddlZdd lmZmZmZdd lmZdd lmZmZmZmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%edurdd l&m'Z'ddl(m)Z)z ddl*Z*Wne+y6dZ*Yn0zddl,m-Z-Wne+y`dZ-Yn0z ddl.Z.Wne+ydZ.Yn0ddl/m0Z0m1Z1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7m8Z8m9Z9m:Z:mZm;Z;mZ?e?@dddgeAe6BZCeCdde6D7ZCedurgZDnddgZDeDdgZEeFedrFeCGeHdddZIddZJddZKd d!ZLGd"d#d#eMZNdTd%d&ZOd'd(ZPdUd)d*ZQd+d,ZRd-d.ZSd/d0ZTd1d2ZUe"d3egd?e"d@gdAdBdCZXe"d=eCe"d>ddDgdEdFZYe"d=eCdGdHZZdIdJZ[ee"dKgdLdMdNZ\ee"d=gdOdPdQZ]dRdSZ^e"dTe6e"dUe6dVdWZ_dXdYZ`edZd[Zad\d]Zbe"d>gd^d_d`Zce"d=eEdadbZdedcddZeeee"degdfe"dgddhgdidjZfe"d=eEdkdlZgee"d=eEdmdnZhee"d=eie?dohe"dpdqdrgdsdtZjee"d=eDdudvZkee"d=e6dwdxZldydzZme"d=e6e"d{dgd|fd}gd~fgddZnee"d=eEddZoddZpee"d=eDddZqddZrddZsddZtee"d=eEejVjuddddZvGddde3ZwddZxe"d=eCddZyee"d=eEe"d>gdddZzddZ{ddZ|e$eduddddZ}ddZ~eEddZeddedDee"d=ee"d3e;egde"d=eEe"d3egdddZddZddZejV"d>dDd}gee"d=eie?dohddZe"d=eCe"d>gdddZddZddZee"d=e?e"dpdqdrge"d>gdddZe"d=e?e"dpdqdrge"d>gdddZe"d=e?e"dpdqdrge"d>gdddZe"d=e?e"ddqdrge"ddqdrge"d>gdd d Ze"d gd e"ddqdrge"ddqdrgd dZeee"d=eDddZe"d=edu rddgngdddZee"d=eDddZee"d3eddDge"dgdddZe"d3egd:d;d<Zeee"d=gd>e"d>dDd6ge"d3edDd6ge"dAgdBe"d3egdEdFdGZ̐dHdIZ͐dXdJdKZee"d>dDd}ge"d=eDe"d3edDd}ge"d=eDdNdOZeejV"d>dDd}gdPdQZeejV"d>dDd}gdRdSZdS(Yz Test the parallel module. N) nullcontextsqrt TimeoutError) PicklingError)sleep)format_exception)dumploadparallel)mp)IS_GIL_DISABLEDnpwith_multiprocessing with_numpy)check_subprocess_call parametrizeraisesskipifwarns)get_reusable_executor)Queue) parallel_sum) LokyBackendMultiprocessingBackendParallelBackendBaseSequentialBackendThreadingBackend) BACKENDSParallel cpu_countdelayedeffective_n_jobsr parallel_backendparallel_configregister_parallel_backendmultiprocessingcCsg|]}t|qS)r ).0Z backend_strr)r)?E:\soccer\ml_env\Lib\site-packages\joblib/test/test_parallel.py Rr,loky threading get_contextspawncCs ttjSN)r r DEFAULT_BACKENDr)r)r)r+get_default_backend_instance^sr4cCst|dt|ddS)N_poolZ_workers)getattrbackendr)r)r+ get_workersfsr9cCs||Sr2r))xyr)r)r+divisionjsr<cCs|dSNr)r:r)r)r+squarensr@c@seZdZdZddZdS)MyExceptionWithFinickyInitz,An exception class with non trivial __init__cCsdSr2r))selfabcdr)r)r+__init__usz#MyExceptionWithFinickyInit.__init__N__name__ __module__ __qualname____doc__rGr)r)r)r+rArsrAFcCs"|dkr|rtddddnt|S)NrCrDrErF)rA ValueError)r:custom_exceptionr)r)r+exception_raiserys rPcCstdtdS)Ng?)timerKeyboardInterruptr?r)r)r+interrupt_raisers rScCs|d||S)zNA module-level function so that it can be spawn with multiprocessing. r>r))r:r;zr)r)r+fsrUcCsttdSNr)typer get_active_backendr)r)r)r+_active_backend_typesrYcCst||dddtdDS)Nn_jobsr8css|]}tt|VqdSr2r#r@r*ir)r)r+ sz parallel_func..r!range) inner_n_jobsr8r)r)r+ parallel_funcsrdcCstdksJdSrV)r"r)r)r)r+test_cpu_countsrecCstdksJdSrVr$r)r)r)r+test_effective_n_jobssrgcontextzbackend_n_jobs, expected_n_jobs)r`r`r[Nr)z positive-intz negative-intNone)ZidscCsT|d|d"tdd|ks JWdn1s40YtdddksPJdS)Nr/rjrrf)rhZbackend_n_jobsZexpected_n_jobsr)r)r+test_effective_n_jobs_Nones 0rmr8r[)rr>riverbose)r> dcCs:ddtdDt|||dddtdDks6JdS)NcSsg|] }t|qSr)r@r*r:r)r)r+r,r-z(test_simple_parallel..r[r8rocss|]}tt|VqdSr2r\rsr)r)r+r_r-z'test_simple_parallel..rbr!)r8r[ror)r)r+test_simple_parallels rwr>c Csd}td}Gdddt}|||dd}|ddt|Dt}|jD],}||}rN|\} } || | qNt |d ksJdS) Nrqz(Done\s+\d+ out of \d+ \|)c@seZdZgZddZdS)z/test_parallel_pretty_print..ParallelLogcSs|j|dSr2)messagesappend)rBmsgr)r)r+_printsz6test_parallel_pretty_print..ParallelLog._printN)rIrJrKrxr{r)r)r)r+ ParallelLogsr|i'rucSsg|]}tt|qSr)r#rUr]r)r)r+r,r-z.test_parallel_pretty_print..r) recompiler!rbsetrxsearchspanaddlen) r8r[n_taskspatternr|executorZlensmessagesrCrDr)r)r+test_parallel_pretty_prints   rcs|jtdddtjdd>}td|ddd td D}|gd ksPJWdn1sd0Yd d |D}|dvst|trdfdd |D}t |dksJdS)NnameZ!some_new_name_for_the_main_thread)targetrvalueTrecordr>rZcss|]}tt|VqdSr2r\rsr)r)r+r_sz6test_main_thread_renamed_no_warning..r`)rrcSs*g|]"}dt|jvrt|jts|qS)zworker timeout)strr isinstanceDeprecationWarningr*wr)r)r+r,s z7test_main_thread_renamed_no_warning..)Nr(z3multi-threaded, use of fork() may lead to deadlockscsg|]}t|jvr|qSr))rrr message_partr)r+r,r-r) setattrr/current_threadwarningscatch_warningsr!rbrrr)r8 monkeypatchwarninforesultsr)rr+#test_main_thread_renamed_no_warnings&. rcCstjdd&}tdt||dWdn1s80Ydd|D}|r|rtdd|D}trzt|d kn t|d k}|o|Sd S|rJdSdS) NTralways)r8rccSsg|] }|jqSr))rrr)r)r+r,r-z*_assert_warning_nested..css|]}d|jdvVqdS)zbacked parallel loops cannotrN)args)r*Zeachr)r)r+r_ sz)_assert_warning_nested..rF)rr simplefilterrdallrr)r8rcexpectedrZwarnings_are_correctZwarnings_have_the_right_lengthr)r)r+_assert_warning_nesteds *rz%parent_backend,child_backend,expected))r.r(T)r.r.F)r(r(T)r(r.T)r/r(T)r/r.Tcsltd|dfddtdDtd|dfddtdD}|dkr\t|shJn t|shJdS)Nr>rZc3s |]}ttdddVqdS)rFr8rcrNr#rr*_ child_backendr)r+r_(sz0test_nested_parallel_warnings..rtc3s |]}ttddVqdS)r>rNrrrrr)r+r_0sr/)r!rbanyr)parent_backendrrresr)rr+test_nested_parallel_warningss r)r.r(r/cCs>dg}dd}tj||fd}|||ds:JdS)NFcSshtjdd,}tddddtdDWdn1s>0Ytt|t|dk|d<dS) NTrr>rjcss|]}ttdVqdS)皙?Nr#rrr)r)r+r_Fr-zPtest_background_thread_parallelism..background_thread..rr)rrr!rbprintr)is_run_parallelrr)r)r+background_threadDs: z=test_background_thread_parallelism..background_thread)rrr)r/Threadstartjoin)r8rrtr)r)r+"test_background_thread_parallelism?s rcCs"td|dddtdDdS)Nr>rZcss|]}ttdVqdS{Gz?Nr\rr)r)r+r_Qr-znested_loop..rar7r)r)r+ nested_loopPsrrrcs&td|dfddtdDdS)Nr>rZc3s|]}ttVqdSr2)r#rrrr)r+r_Wsz#test_nested_loop..ra)rrr)rr+test_nested_loopTsrcCstdSr2)rNr7r)r)r+raise_exception\src CstttXtddd,}|ttdttdgWdn1sH0YWdn1sf0YdS)Nr>r.rZ)rrNr!r#rr)r r)r)r+)test_nested_loop_with_exception_with_loky`s rcs<tddtdddfddtdDs8JdS) z1Input is mutable when using the threading backendrt)maxsizer>r/rZc3s|]}tjdVqdSrN)r#putrqr)r+r_jr-z1test_mutate_input_with_threads..N)rr!rbfullr)r)rr+test_mutate_input_with_threadsgs "r)rr>r`cCs6td}dd|Dt|ddd|Dks2JdS)z.Check the keyword argument processing of pmap. cSsg|]}t|ddqSrr;rUrsr)r)r+r,rr-z(test_parallel_kwargs..rjcss|]}tt|ddVqdSrrNr}rsr)r)r+r_rsz'test_parallel_kwargs..Nrv)r[lstr)r)r+test_parallel_kwargsnsrcCstd}dd|D}td|dh}|j}||dd|DksDJ||dd|Dks^Jtdur|t|t|jus|JWdn1s0Ytdurt|jdusJ||d d|DksJtdurt|jdusJdS) NrcSsg|]}t|ddqSrrrsr)r)r+r,zr-z4test_parallel_as_context_manager..rrZcss|]}tt|ddVqdSrr}rsr)r)r+r_r-z3test_parallel_as_context_manager..css|]}tt|ddVqdSrr}rsr)r)r+r_r-css|]}tt|ddVqdSrr}rsr)r)r+r_r-)rbr!_backendr r9)r8rrpZmanaged_backendr)r)r+ test_parallel_as_context_managerws4rcsbGdddtttdd2tdddfdd td DWd n1sT0Yd S) z`Check that pmap captures the errors when it is passed an object that cannot be pickled. c@seZdZddZdS)z1test_parallel_pickling..UnpicklableObjectcSs tddS)NZ123) RuntimeErrorrBr)r)r+ __reduce__sz.UnpicklableObject.__reduce__N)rIrJrKrr)r)r)r+UnpicklableObjectsrzthe task to sendmatchr>r.rZc3s|]}ttVqdSr2r#idrrr)r+r_sz)test_parallel_pickling..rN)objectrrr!rbr)r)rr+test_parallel_picklings r byteorder)<>= max_nbytesZ1Mcsddtdd|djj}tdd|dfd d td D}|D]2\}}||ksjJ||jjkszJtj |qVdS) NcSs ||jjfSr2)dtyperr?r)r)r+inspect_byteordersz=test_parallel_byteorder_corruption..inspect_byteorder)r>r`i4r>r.)r[r8rc3s|]}tVqdSr2r#rrr:r)r+r_sz5test_parallel_byteorder_corruption..r`) rZarangeZreshapeviewrrr!rbtestingassert_array_equal)rrZinitial_np_byteorderresultZ x_returnedZbyteorder_in_workerr)rr+"test_parallel_byteorder_corruptions  rcCs0ttd|ddddtdDdks,JdS)Nr>r[r8timeoutcss|]}ttdVqdS)gMbP?Nrrsr)r)r+r_sz0test_parallel_timeout_success..r)rr!rbr7r)r)r+test_parallel_timeout_successsrcCsLtt0td|ddddtdDWdn1s>0YdS)Nr>rrcss|]}ttdVqdSrNrrsr)r)r+r_sz-test_parallel_timeout_fail..r)rrr!rbr7r)r)r+test_parallel_timeout_fails r sequential return_as generatorgenerator_unorderedcCsxtt6ttd||ddddtdDWdn1sD0Yttd||ddddtdDdS)Nr>r)r[r8rrcss|]}ttdVqdSrrrsr)r)r+r_sz.rcss|]}ttdVqdSrrrsr)r)r+r_s)rrlistr!rbr8rr)r)r+)test_parallel_timeout_fail_with_generators "rc CstdurrZcSsg|]\}}tt||qSr)r#r<r*r:r;r)r)r+r,r-z&test_error_capture..)rr)rrcSsg|]}tt|qSr)r#rSrsr)r)r+r,r-cSsg|]\}}tt||qSr)rrr)r)r+r,r-cSsg|]}t|ddqSrrrsr)r)r+r,r-rcss|]}tt|ddVqdSrr}rsr)r)r+r_sz%test_error_capture..cSsg|]}tt|qSr)rrsr)r)r+r, r-cSsg|]}t|ddqSrrrsr)r)r+r,r-css|]}tt|ddVqdSrr}rsr)r)r+r_srjcSsg|]}tt|qSr)rrsr)r)r+r,"r-cSsg|]\}}tt||qSr)rrr)r)r+r,'r-r)r[rocss|]}tt|ddVqdS)T)rONr#rPr]r)r)r+r_+r-r)r rZeroDivisionErrorr!ziprRr9rrb _iteratingZn_completed_tasksZn_dispatched_tasks _abortingrA)r8r Zoriginal_workersr)r)r+test_error_capturesT   "   "  6  2$  8 >  rc Csddd}td|d}ttdd(|dd |dd DWdn1sR0Yttdd(|d d |d d DWdn1s0Yttdd(|d d |dd DWdn1s0YWdn1s0YdS)Nrcss(tdD]}||krtd|VqdS)NIterator Raising Error)rbrN)raise_atr^r)r)r+ my_generator2s z1test_error_in_task_iterator..my_generatorr>rZrrcss|]}tt|VqdSr2r\r]r)r)r+r_;r-z.test_error_in_task_iterator..)rcss|]}tt|VqdSr2r\r]r)r)r+r_@r-rtcss|]}tt|VqdSr2r\r]r)r)r+r_Dr-)r)r!rrN)r8rrr)r)r+test_error_in_task_iterator/s 66rcCs|d|dS)Nz Consumed %s)ry)queueitemr)r)r+consumerGsr zbatch_size, expected_queue) Produced 0 Consumed 0 Produced 1 Consumed 1 Produced 2 Consumed 2 Produced 3 Consumed 3 Produced 4 Consumed 4 Produced 5 Consumed 5r) r rrrr rrrrrrrcsTtfdd}td||dfdd|D|ks@JtdksPJdS) z=Test that with only one job, Parallel does act as a iterator.c3s&tdD]}d||VqdSNrz Produced %irbryr^r r)r+producerxs z'test_dispatch_one_job..producerrr[ batch_sizer8c3s|]}tt|VqdSr2r#r rsrr)r+r_}sz(test_dispatch_one_job.. N)rr!r)r8rZexpected_queuerr)rr+test_dispatch_one_jobKs+  r!cst}|fdd}tddd|dfdd|Dt}|d d ksXJ|d d d }|dksvJ|d}||ksJtdksJd S)zRCheck that using pre_dispatch Parallel does indeed dispatch items lazily. c3s&tdD]}d||VqdSrrrrr)r+rs z/test_dispatch_multiprocessing..producerr>rr`)r[r pre_dispatchr8c3s|]}ttdVqdS)rNrrrr)r+r_sz0test_dispatch_multiprocessing..rr Nrz Consumed anyrirr )r Managerrr!indexr)r8managerrZqueue_contentsZfirst_consumption_indexZproduced_3_indexr)rr+test_dispatch_multiprocessings    r&cCsZtdddd8}|ddtdD|jdks8JWdn1sL0YdS) Nr>autor/rcss|]}tt|VqdSr2rr]r)r)r+r_r-z/test_batching_auto_threading..rr!rbrZcompute_batch_size)rr)r)r+test_batching_auto_threadingsr*cCsZtdd|d8}|ddtdD|jdks8JWdn1sL0YdS)Nr>r'rcss|]}tt|VqdSr2rr]r)r)r+r_r-z2test_batching_auto_subprocesses..r(rr))r8rr)r)r+test_batching_auto_subprocessessr+cCsLtt0tddddddtdDWdn1s>0YdS) zCMake sure that exception raised during dispatch are indeed capturedr>r)r[r"rocss|]}tt|VqdSr2rr]r)r)r+r_sz*test_exception_dispatch..rN)rrNr!rbr)r)r)r+test_exception_dispatchs r-cCs tddddtdDdS)Nr>rjcss|]}tt|VqdSr2rr*jr)r)r+r_r-z(nested_function_inner..rrarr)r)r+nested_function_innersr0cCs tddddtdDdS)Nr>rjcss|]}tt|VqdSr2)r#r0r.r)r)r+r_r-z(nested_function_outer..rrarr)r)r+nested_function_outersr1z'https://github.com/joblib/loky/pull/255reasoncCstt.}td|dddtdDWdn1s<0Yt|j|j|j}d|}d|vsnJd |vszJd |vsJt|jtusJdS) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. r>rZcss|]}tt|VqdSr2)r#r1r]r)r)r+r_sz1test_nested_exception_dispatch..rNr1r0rP) rrNr!rbr rWrtbr)r8excinfoZ report_linesreportr)r)r+test_nested_exception_dispatchs &    r8c@s$eZdZdZdddZd ddZdS) FakeParallelBackendz8Pretends to run concurrently while running sequentially.rNcKs|||_||_|Sr2)r$r[r )rBr[r Z backend_argsr)r)r+ configures zFakeParallelBackend.configurecCs"|dkrttd|d}|S)Nrr)maxr r"rBr[r)r)r+r$sz$FakeParallelBackend.effective_n_jobs)rN)r)rIrJrKrLr:r$r)r)r)r+r9s r9c CsttddtddWdn1s,0Yttdd:tddWdn1sd0YWdn1s0Yttdd:tddWdn1s0YWdn1s0YdS)NzInvalid backend:rz unit-testingr7)rrNr!r&r)r)r)r+test_invalid_backends( < r=cCs0tt }td|dWdn1s.0Ydt|jvsJJtt }td|dWdn1sx0Ydt|jvsJtt }td|dWdn1s0Ydt|jvsJtt }td|dWdn1s0Ydt|jvs,JdS)NrrZz&n_jobs == 0 in Parallel has no meaning?z2.3z$n_jobs could not be converted to intZ invalid_str)rrNr!Z_initialize_backendrr)r8r6r)r)r+test_invalid_njobss . . . 0r?)2gffffff@r>cCsPt||d}|dksJ|ddtdD}tddt|DsLJdS)NrZr>css|]}tt|VqdSr2r\r]r)r)r+r_r-z.test_njobs_converted_to_int..rcss|]\}}|t|kVqdSr2rr)r*r^rr)r)r+r_r-)r!_effective_n_jobsrbr enumerate)r8r[rrr)r)r+test_njobs_converted_to_ints rDcCs>z0tdtdtvsJtdtks(JWtd=ntd=0dS)NZ test_backend)r'r9r r)r)r)r+test_register_parallel_backends   rEcCs^tj}ttksJz*tdtdddttks8JW|t_n|t_0ttksZJdS)Nr/T)Z make_default)r r3rYr4r'r r)Zdefault_backend_origr)r)r+test_overwrite_default_backend%srFzOnly without multiprocessingcCsttdd,tddddtdDWdn1s>0Ytdd(tddtdDWdn1s0YdS) Nz)joblib backend '.*' is not available on.*rr.r7css|]}tt|VqdSr2r\r]r)r)r+r_4r-z2test_backend_no_multiprocessing..r`css|]}tt|VqdSr2r\r]r)r)r+r_8r-)r UserWarningr!rbr&r)r)r)r+test_backend_no_multiprocessing1s: rHcCs||ddt\}}|dks&Jtddks6Jt}|jdksJJ|dkrvt|tusbJt|jtusJn|dkrt|tusJt|jtusJnX|dkrt|t usJt|jt usJn,| drt|t usJt|jt usJWdn1s0YdS)Nr`rjr(r.r/Ztest_) r rXr$r!r[rWrrrr startswithr9)rh backend_nameactive_backend active_n_jobsrr)r)r+check_backend_context_manager;s$   rMcCsg|] }d|qS)ztest_backend_%dr)r]r)r)r+r,Qr-r`csr|tvr|t|tttks&Jt|ttks@JtdddfddtDttksnJdS)Nr>r/rZc3s |]}|stt|VqdSr2)r#rM)r*rDrhr)r+r_csz/test_backend_context_manager..)r setitemr9rYr4rMr! all_backends_for_context_manager)rr8rhr)rNr+test_backend_context_managerTs rQc@seZdZdZdddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.NcCs|durtd||_dS)Nzparam should not be None)rNparam)rBrSr)r)r+rGpsz%ParameterizedParallelBackend.__init__)NrHr)r)r)r+rRmsrRcCs|tdtttksJ|dddd~t\}}t|tusJJ|jdksXJ|dksdJt }|j dksxJ|j |usJ|ddt dD}Wdn1s0Y|dd t dDksJttksJdS) NZ param_backend*r`)rSr[css|]}tt|VqdSr2r#rr]r)r)r+r_r-z=test_parameterized_backend_context_manager..rtcSsg|] }t|qSr)rr]r)r)r+r,r-z>test_parameterized_backend_context_manager..) rOr rRrYr4r rXrWrSr!r[rrb)rrhrKrLrrr)r)r+*test_parameterized_backend_context_managervs  4rVcCsttksJ|tdddd~t\}}t|tus@J|jdksNJ|dksZJt}|jdksnJ|j |us|J|ddt dD}Wdn1s0Y|ddt dDksJttksJdS) N+)rSrtrjcss|]}tt|VqdSr2rUr]r)r)r+r_r-zFtest_directly_parameterized_backend_context_manager..cSsg|] }t|qSr)rr]r)r)r+r,r-zGtest_directly_parameterized_backend_context_manager..) rYr4rRr rXrWrSr!r[rrb)rhrKrLrrr)r)r+3test_directly_parameterized_backend_context_managers  4rXcCstdtS)Nr)rosgetpidr)r)r)r+sleep_and_return_pidsr[cCs<ttksJtdks JtddddtdDS)Nrr>rjcss|]}ttVqdSr2)r#r[rr)r)r+r_r-z"get_nested_pids..)rYrr!rBrbr)r)r)r+get_nested_pidssr\cs eZdZdZfddZZS) MyBackendz:Backend to test backward compatibility with older backendscstt|dSrV)superr]get_nested_backendr __class__r)r+r_szMyBackend.get_nested_backend)rIrJrKrLr_ __classcell__r)r)r`r+r]sr]back_compat_backend)r/r.r(rccCsf||JtddddtdD}|D]}tt|dks*Jq*Wdn1sX0YdS)Nr>rjcss|]}ttVqdSr2)r#r\rr)r)r+r_r-z6test_nested_backend_context_manager..rr)r!rbrr)rhr8Z pid_groupsZ pid_groupr)r)r+#test_nested_backend_context_managers rd)r>riNcs|ddtddfddtdD|d4tddfddtdDWdn1sn0YdS) NcSs0tt|ksJt|}t|ks,JdSr2)rYr r$r!rB)Zexpected_backend_typeZexpected_n_jobr)r)r+check_nested_backendsz?test_nested_backend_in_sequential..check_nested_backendrrjc3s|]}ttjdVqdSr)r#r r3r)rer)r+r_sz4test_nested_backend_in_sequential..rc3s|]}tVqdSr2rrr8rer[r)r+r_sra)r8r[rhr)rfr+!test_nested_backend_in_sequentials rgcCsZ||>}|tur|d}|tur*|d}|j|ks8JWdn1sL0YdS)Nr8r)r&r% nesting_level)rh inner_backendZexpected_levelctxr8r)r)r+check_nesting_levels  rk outer_backendrics~t|dtd|dfddtdDdd.tfddtdDWdn1sp0YdS) Nrr>rZc3s|]}ttdVqdSrr#rkrrhrir)r+r_sz-test_backend_nesting_level..rrjc3s|]}ttdVqdSrrmrrnr)r+r_s)rkr!rb)rhrlrir)rnr+test_backend_nesting_levels rowith_retrieve_callbackTcsddlGfdddt}td|dd|dd}tddfd d td D|turt|d jd kstJ|tur|djd ksJWdn1s0YdS)Nrcs"eZdZdZZjddZdS)z)test_retrieval_context..MyBackendrcss|jd7_dVdSrkrrr)r)r+retrieval_context sz;test_retrieval_context..MyBackend.retrieval_contextN)rIrJrKr^Zsupports_retrieve_callbackcontextmanagerrqr)) contextlibrpr)r+r]sr]Z retrievalcSstddddt|DS)Nr>rjcss|]}tt|VqdSr2rr]r)r)r+r_r-z>test_retrieval_context..nested_call..ra)nr)r)r+ nested_callsz+test_retrieval_context..nested_callr>rjc3s|]}t|VqdSr2rr])rur)r+r_r-z)test_retrieval_context..rtr8r)rsrr'r!rbr&r^r%)rhrpr]rjr))rsrurpr+test_retrieval_contexts   rvr)rrigQ?cCs6ttt|dWdn1s(0YdS)Nr)rrNr!rwr)r)r+test_invalid_batch_sizes rxz)n_tasks, n_jobs, pre_dispatch, batch_size) )r>r>rr')r>r>r[r'rr>r[r')ir>r[r'ry)rrr[r')r r[r')r  2 * n_jobsr)r rr)r}r r|rM)rzr r|r'cCsN|||d}ddt|D}tfi|ddt|D}||ksJJdS)N)r[r"rcSsg|] }t|qSr)rrr]r)r)r+r,:r-z0test_dispatch_race_condition..css|]}tt|VqdSr2r\r]r)r)r+r_;r-z/test_dispatch_race_condition..rv)rr[r"rparamsrrr)r)r+test_dispatch_race_condition%s  rcCs8t}tddd}|jd}|}||ks4JdS)Nr>r(rZrh)r get_start_methodr!Z_backend_kwargsget)Zmp_start_methodrrhZ start_methodr)r)r+test_default_mp_context?s   rcs^|dkrtd}tjd}|ddtjtd|dfddt dDdS) Nr(r1rTir>rZc3s |]}ttjjVqdSr2)r#rdotTr]rCr)r+r_^r-zAtest_no_blas_crash_or_freeze_with_subprocesses..) r r0rrandomZ RandomStaterandnrrr!rb)r8rngr)rr+.test_no_blas_crash_or_freeze_with_subprocessesHs    rafrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) cCsB|dkrtdkrtdt|}ttjd|gddddS)Nr(ZforkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r stdout_regex) r rpytestskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrsys executable)r8coder)r)r+2test_parallel_with_interactively_defined_functionsqs raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_position)r#rkwargsc Csv|dvr$|tkstjdkr$tdtj|||tj tj t j d}| d}| |ttj|jgddddS) N)r(r1win32zNot picklable with pickle)rr8rjoblib_root_folderzunpicklable_func_script.pyrrr) SQUARE_MAINrplatformrr)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrrYpathdirnamejoblib__file__rwriterrstrpath)r8rrtmpdirrZ code_filer)r)r+0test_parallel_with_unpicklable_functions_in_argss&    raimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(backend="loky", n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )rcCs.|d}|tttj|jgddddS)Nz(joblib_interactively_defined_function.pyr)rr)rr5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrrrrscriptr)r)r+7test_parallel_with_interactively_defined_functions_lokys   raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(backend="loky", n_jobs=2)( delayed(l.append)(i) for i in range(3) )) cCs0|d}|tttj|jgdddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\]Z LokyProcess)rZ stderr_regexr)rr9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrrrrr)r)r+:test_parallel_with_interactively_defined_bound_method_lokys   rcCs"tg}tdd|gksJdS)Nr>rj)iterr!)Zexhausted_iteratorr)r)r+%test_parallel_with_exhausted_iterator)srcCs"t|tjstdt||S)Nz#Expected np.memmap instance, got %r)rrmemmap TypeErrorrWcopyrr)r)r+ check_memmap.s rcCsdd}tdd|ddd|dD}t||t|D]\}}tj||q:td d|dd d|dD}t||t|D]\}}tj||qdS) Ncss(t|D]}tjdtjd|VqdS)Nrr)rbronesZfloat32)rtr^r)r)r+generate_arrays<s zBtest_auto_memmap_on_arrays_from_generator..generate_arraysr>rr[rr8css|]}tt|VqdSr2r#rr*rCr)r)r+r_Bsz.rqrcss|]}tt|VqdSr2rrr)r)r+r_Ks)r!rrrrr)r8rrrrr)r)r+)test_auto_memmap_on_arrays_from_generator4srcCs|Sr2r))argr)r)r+identityRsrcs|dj}tj}tj|ddtj|ddg}t||t|ddt ddfdd d D\}t d tj svJd j |ksJtj ||dS) Nz test.mmapZuint8rrA)Z mmap_moder>rjc3s|]}ttVqdSr2)r#rrrr)r+r_^r-z.test_memmap_with_big_offset..)rr)rrmmapALLOCATIONGRANULARITYrZzerosrr r r!rroffsetrr)rfnamesizeobjrr)rr+test_memmap_with_big_offsetVs   rcCstjdd.}tdddddtdDWdn1s@0Yt|dksZJ|d}t|jtsrJt|jd ksJdS) NTrr)r[rcss|]}tt|VqdSr2r\r]r)r)r+r_fr-zFtest_warning_about_timeout_not_supported_by_backend..2rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) rrr!rbrrrrGr)rrr)r)r+3test_warning_about_timeout_not_supported_by_backendds<rcCs |||<|Sr2r)) input_listr$rr)r)r+set_list_valueqsr)rr>rcsVdgdt|dddfddtdD}t|}tddt|DsRJdS) Nrrtrr/r[rr8c3s|]}tt||VqdSr2)r#rr]rr)r+r_|szPtest_parallel_return_order_with_return_as_generator_parameter..css|]\}}||kVqdSr2r)r*vrAr)r)r+r_r-)r!rbrrr)r[rr)rr+=test_parallel_return_order_with_return_as_generator_parametervs  rcCs|r tdt|S)Nr)rr)edelayr)r)r+_sqrt_with_delaysrcspt|d|dddtdDtfddtdD}dgttd d}td dt||DsjJdS) Nrrcss$|]}tt|d|dkVqdS)r>rN)r#rr]r)r)r+r_szK_test_parallel_unordered_generator_returns_fastest_first..rc3s|]}tVqdSr2)nextrrr)r+r_r- rr>css|]\}}||kVqdSr2r)rr)r)r+r_r-)r!rbsortedrrr)r8r[Zquickly_returnedZexpected_quickly_returnedr)rr+8_test_parallel_unordered_generator_returns_fastest_firstsrcCst||dSr2)rr8r[r)r)r+7test_parallel_unordered_generator_returns_fastest_firstsr)rr>rnricCsvdgdgd}tt2t}t||ddd|DWdn1sP0Yt|}|dksrJdS)NrCrrqrZcss|]}ttj|VqdSr2r#rQrr]r)r)r+r_r-z%test_abort_backend..r)rrrQr!)r[r8Zdelayst_startdtr)r)r+test_abort_backends  8 rcCstjtdtd}d|d<|S)NgArFr)rrintbool)rrr)r)r+get_large_objectsrcCsZt|||d8}|ddtdD}t|t|~Wdn1sL0YdS)N)r[r8rcss|]}tt|VqdSr2)r#rr]r)r)r+r_r-z0_test_deadlock_with_generator..r)r!rbr)r8rr[r rr)r)r+_test_deadlock_with_generators rcCst|||dSr2)r)r8rr[r)r)r+test_deadlock_with_generatorsrcCsttddRt|||d}|ddtdD}t}|ddtdD}Wdn1sd0Yt|d ksJd ~dS) N)This Parallel instance is already runningrrcss|]}ttdVqdSrrrr)r)r+r_r-z/test_multiple_generator_call..rcss|]}tt|VqdSr2rr]r)r)r+r_r-rqr>[The error should be raised immediately when submitting a new task but it took more than 2s.)rrr!rbrQ)r8rr[r grZgen2r)r)r+test_multiple_generator_calls4rc Cst|||d}|ddtdD}t}ttdd&|ddtdD}Wdn1sf0Yt|d ksJd Wdn1s0Y~dS) Nrcss|]}ttdVqdSrrrr)r)r+r_r-z7test_multiple_generator_call_managed..rrrcss|]}tt|VqdSr2rr]r)r)r+r_r-rqr>r)r!rbrQrr)r8rr[r rrg2r)r)r+$test_multiple_generator_call_manageds4"r return_as_1 return_as_2cCst|||dddtdD}t|||dddtddD}|dkrRt|}|dkrbt|}tddt|tdDsJtd dt|tddDsJdS) Nrcss|]}tt|dVqdSr>NrUr]r)r)r+r_sz9test_multiple_generator_call_separated..rcss|]}tt|dVqdSrrUr]r)r)r+r_srrcss|]\}}||kVqdSr2r)r*rr^r)r)r+r_r-css|]\}}||kVqdSr2r)rr)r)r+r_r-)r!rbrrr)r8rrr[rrr)r)r+&test_multiple_generator_call_separateds rzbackend, error))r.T)r/F)rFc CsV|dkrtdurtdtd||d}|ddtdD}t|dd }|r^ttd d nt }|lt }td||dd dtdd D}|dkrt |}t ddt |tdd DsJWdn1s0Yt |dksJd} |jr"| dkr"| d7} t dqt |dks8J|dkrR|jsRJdS)Nr.zRequires multiprocessingr>rcss|]}ttdVqdSrrr]r)r)r+r_&r-z.rcSstdS)NzGenerator collected)rr)r)r)r+'r-z;test_multiple_generator_call_separated_gc..z The executor underlying Parallelrcss|]}tt|dVqdSrrUr]r)r)r+r_3srrcss|]\}}||kVqdSr2r)rr)r)r+r_:r-rtrr`rr>)r rrr!rbweakreffinalizerrrrQrrraliverrBr) r8rrerrorr rZg_wrrjrretryr)r)r+)test_multiple_generator_call_separated_gcs2  @ rcCs|j}tdd||dF}|ddtjdgdDtt|dksNJWdn1sb0YtdD]}t|sqtd qtt d tdd|d }|d dtjdgdDtdD]}t|sqtd qt d dS) Nr>r)r[rr8Z temp_foldercss|]}tt|VqdSr2rrr)r)r+r_Vr-z(test_memmapping_leaks..rrrqrz/temporary directory of Parallel was not removedrcss|]}tt|VqdSr2rrr)r)r+r_fr-) rr!rrrrYlistdirrbrAssertionError)r8rrrr)r)r+test_memmapping_leaksKs  4       r)Nr.r/cCs<td|dddtdD}|ddtdDks8JdS)Nr>rZcss|]}tdd|VqdS)cSs|dSr=r)r?r)r)r+rvr-z2test_lambda_expression...Nrr]r)r)r+r_usz)test_lambda_expression..rcSsg|] }|dqSr>r)r]r)r)r+r,xr-z*test_lambda_expression..ra)r8rr)r)r+test_lambda_expressionpsrcsd}d}d|td||d}|fddt|D|jj|jjksLJ|jj|jjks`J|fddt|D|jj|jjksJ|jj|jjksJd S) zCTest that a parallel backend correctly resets its batch statistics.r>ig@r)ror[r8c3s|]}ttjVqdSr2rr] task_timer)r+r_r-z6test_backend_batch_statistics_reset..c3s|]}ttjVqdSr2rr]rr)r+r_r-N)r!rbrZ_effective_batch_sizeZ_DEFAULT_EFFECTIVE_BATCH_SIZEZ_smoothed_batch_durationZ _DEFAULT_SMOOTHED_BATCH_DURATION)r8r[Zn_inputsrr)rr+#test_backend_batch_statistics_reset{s rcCsdD]x}tt|djtus"Jt|dd}t|jtus@Jt|dd}t|jtus^Jt|dd}t|jtusJqtdd dd }t|jtusJ|d dd:tdd }t|jtusJ|jdksJWdn1s0Y|d dd@td dd}t|jtus"J|jd ks2JWdn1sH0Y|d dd>tdd }t|jtus~J|jdksJWdn1s0Y|d dd@td dd}t|jtusJ|jd ksJWdn1s0YdS)N)rr>rirjthreadsr[prefer processes sharedmemr[requirer>r.)r[r8rrr`rr)rWr!rr4rrr[)rhr[rr)r)r+$test_backend_hinting_and_constraintss4    , 0 0 rr)NrrcCsddtdD}t||dddtdD}||ks.rrcss|]}tt|VqdSr2r\r]r)r)r+r_sz6test_backend_hinting_always_running..)rr[css|]}tt|VqdSr2r\r]r)r)r+r_r-)rbr!r&)r[rZexpected_resultsrr)r)r+#test_backend_hinting_always_runnings 6rcCsGdddt}||Ltddd}t|j|us:Jtddd}t|j|usXJWdn1sl0YGdd d t}||tddd}t|j|usJ|\}}|d ksJ|d ksJtddd d }t|jtusJ|\}}d }||ksJ|d ks$JWdn1s:0Yttt|ddWdn1sr0YdS)Nc@s$eZdZdZdZddZddZdS)z[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcSsdSr2r)rr)r)r+ apply_asyncszgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_asynccSs|Sr2r)r<r)r)r+r$szltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsNrIrJrKZsupports_sharedmemZ use_threadsrr$r)r)r)r+MyCustomThreadingBackendsrr>rrrrc@s$eZdZdZdZddZddZdS)z\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcSsdSr2r)rr)r)r+rszhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asynccSs|Sr2r)r<r)r)r+r$szmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsNrr)r)r)r+MyCustomProcessingBackendsrr4r)r[rrozUsing ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r8r) rr!rWrZ readouterrrstriprrN)Zcapsysrhrrrouterrrr)r)r+9test_backend_hinting_and_constraints_with_custom_backendss,   0      . r cCstttddWdn1s(0YtttddWdn1sZ0YtttdddWdn1s0Ytdur tttdddWdn1s0Ytttd ddWdn1s0YdS) Ninvalidrrrr)rrr.rr()rrNr!r r)r)r)r+,test_invalid_backend_hinting_and_constraints s ( ( *  * rc stddh}t|jj|jjfg}dkr<|WdS|fddtdD}||dWdS1sx0YdS)zCPerform nested parallel calls and introspect the backend on the wayr>rjrNc3s(|] }ttfddiVqdS)limitrN)r#_recursive_backend_infor]rrr)r+r_'sz*_recursive_backend_info..r)r!rWrrIrhrb)rrrZ this_levelrr)rr+r s rcCs^||ddt}Wdn1s(0Y|d}|dfdddg}||ksZJdS)Nr>rjBackendr)rr)rr>)rtitle)rhr8Zbackend_types_and_levelsZtop_level_backend_typeZexpected_types_and_levelsr)r)r+test_nested_parallelism_limit.s$ rcCstddtdDS)z6A horrible function that does recursive parallel callscss|]}ttVqdSr2)r#_recursive_parallelr]r)r)r+r_Ar-z&_recursive_parallel..r>ra)Z nesting_limitr)r)r+r?src Cs||dd>tt}tWdn1s20YWdn1sP0Y|j}|dkrddlm}t||tfrt dt|t sJdS)Nr>rjr.r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r BaseExceptionrrZ&joblib.externals.loky.process_executorrrrrxfailRecursionError)rhr8r6excrr)r)r+test_thread_bomb_mitigationDs B  rcCs*i}dD]}tj|||<q|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSZVECLIB_MAXIMUM_THREADSZNUMEXPR_NUM_THREADSZNUMBA_NUM_THREADS ENABLE_IPCrq)rYenvironrr)Zenv_varsvarr)r)r+_run_parallel_sum`s r"zNeed OpenMP helper compiledcCstd|dddtdD}ttdd}|D]Z\}}||ksFJ|D]<\}}|drr|t|ksJqN|dks~J|dksNJqNq2dS) Nr>rZcss|]}ttVqdSr2)r#r"rr)r)r+r_rsz-test_parallel_thread_limit..rZ_THREADSr1)r!rbr;r"itemsendswithr)r8rZexpected_num_threadsZworker_env_varsZomp_num_threadsrrr)r)r+test_parallel_thread_limitos    r&zThis test requires daskcCs8ttdd|dWdn1s*0YdS)NzPlease install daskrZdask)rrNrNr)r)r+)test_dask_backend_when_dask_not_installedsr'c CsGdddt}d}||Ztjt|d,tddddtdDWdn1s`0YWdn1s~0YdS) Nc@s&eZdZddZdddZddZdS) z3test_zero_worker_backend..ZeroWorkerBackendc_sdSrVr))rBrrr)r)r+r:sz=test_zero_worker_backend..ZeroWorkerBackend.configureNcSs tddS)NzNo worker availabler)rBfunccallbackr)r)r+rsz?test_zero_worker_backend..ZeroWorkerBackend.apply_asynccSsdSrVr)r<r)r)r+r$szDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobs)N)rIrJrKr:rr$r)r)r)r+ZeroWorkerBackends r*z&ZeroWorkerBackend has no active workerrr>rjcss|]}tt|VqdSr2rr]r)r)r+r_r-z+test_zero_worker_backend..)rrrrr!rb)rhr*Z expected_msgr)r)r+test_zero_worker_backends   r+csdadddksJtddfddtdD}t|dhksLJdadks^Jtddfd dtdD}t|dhksJdS) Nzoriginal valuecSstSr2)MY_GLOBAL_VARIABLEr)r)r)r+ check_globalssz@test_globals_update_at_each_parallel_call..check_globalsr>rjc3s|]}tVqdSr2rr]r-r)r+r_sz.z changed valuec3s|]}tVqdSr2rr]r.r)r+r_s)r,r!rbr)Zworkers_global_variabler)r.r+)test_globals_update_at_each_parallel_callsr/cCs4ddl}|jdd}|||td}|S)Nrrq threadpoolctl)numpyrrrrZ importorskipZthreadpool_info)rrCr0r)r)r+_check_numpy_threadpool_limitss   r2cCs8|D] }|d|dkr|dSqtd|dS)Nfilepath num_threadsz,An unexpected module was loaded in child: {})rNr) child_module parent_info parent_moduler)r)r+_parent_max_num_threads_fors r8cCs@|D]6}|D],}t||}t|||h}|d|vs Jq qdS)Nr4)r8min)Z workers_infor6r4Zchild_threadpool_infor5Zparent_max_num_threadsrr)r)r+check_child_num_threadssr:)r>rrnricCszt}t|dkrtjddtd|dddtdD}t|}|d krZ|dd }ntt|d }t |||dS) Nr&Need a version of numpy linked to BLASr2r.rcss|]}ttVqdSr2r#r2r]r)r)r+r_sz;test_threadpool_limitation_in_child_loky..r>rr4 r2rrrr!rbr$r;r"r:)r[r6workers_threadpool_infosexpected_child_num_threadsr)r)r+(test_threadpool_limitation_in_child_lokys  r@inner_max_num_threads)rr>rNcCst}t|dkrtjdd|d|d,t|dddtd D}Wdn1s\0Yt|}|d kr|dd }n|durtt|d }n|}t |||dS) Nrr;r2r.rArjcss|]}ttVqdSr2r<r]r)r)r+r_ sz>test_threadpool_limitation_in_child_context..r>rr4r=)rhr[rAr6r>r?r)r)r++test_threadpool_limitation_in_child_contexts   &rCvar_name)rrrc st|dkrtdtddddtj}zdtj<t|dfd d t d D}|ddgksvJ|d dd 2t|dfdd t d D}Wdn1s0Y|ddgksJW|durtj=n |tj<n |durtj=n |tj<0dS)NrzSkip test when n_jobs == 1TZreusecSs tj|Sr2)rYr r)rDr)r)r+_get_env,sz>test_threadpool_limitation_in_child_override.._get_env4rjc3s|]}tVqdSr2rr]rFrDr)r+r_3r-z?test_threadpool_limitation_in_child_override..r>r.rBc3s|]}tVqdSr2rr]rHr)r+r_7sr#) r$rrrshutdownrYr rr!rb)rhr[rDZoriginal_var_valuerr)rHr+,test_threadpool_limitation_in_child_overrides(    "&   rJ)r>rricCsJdd}||tdd}tdD]"}||tdd}||ks"Jq"dS)Ncs,tdt|dfddtdDdS)Nrrjc3s|]}ttVqdSr2)r#sumr]r?r)r+r_Kr-zAtest_loky_reuse_workers..parallel_call..rvrjr)r?r+ parallel_callIsz.test_loky_reuse_workers..parallel_callTrEr)rrb)r[rLZfirst_executorrrr)r)r+test_loky_reuse_workersCs   rMcCsd|t<dS)N initialized)rYrZ)statusr)r)r+_set_initializedYsrPcCst}||d}|dvs*Jd||s2dSd||<td}tdd|D}t|kr||krtdtdd|D}q\t|krtd |S) N)rNstartedz2worker should have been in initialized state, got rQrcSsg|]\}}|dkr|qSrQr)r*pidrr)r)r+r,ir-z!_check_status..rcSsg|]\}}|dkr|qSrRr)rSr)r)r+r,lr-z-Waited more than 30s to start all the workers)rYrZrrQrr$rr)rOr[ wait_workersrTstatedeadlineZ n_startedr)r)r+ _check_status]s      rXcsdt}|||tfd.tfddtdDWdn1sV0YdS)Nr8r[Z initializerZinitargsc3s|]}ttVqdSr2r#rXr]r[rOr)r+r_r-z+test_initializer_context..rq)r r#dictrPr!rb)r[r8rhr%r)r[r+test_initializer_contexttsr]cs>t}|t|tfdfddtdDdS)NrYc3s|]}ttVqdSr2rZr]r[r)r+r_r-z,test_initializer_parallel..rq)r r#r\r!rPrb)r[r8r%r)r[r+test_initializer_parallelsr^csxd}t}|t}t|D]<}tdtfdfddtD}|t|}q"t|kstJddS)Nr`r.rYc3s |]}ttddVqdST)rUNrZr]r[r)r+r_sz*test_initializer_reused..z=The workers should be reused when the initializer is the same) r r#r\rrbr!rPunionrr[Z n_repetitionsr%Zpidsr^rr)r[r+test_initializer_reuseds&   rbcs|d}t}t}t|D]D}|tdtfdfddtD}|t|}qt||ksxJddS)Nr`r.rYc3s |]}ttddVqdSr_rZr]r[r)r+r_sz.test_initializer_not_reused..zFThe workers should not be reused when the initializer arguments change) r r#rrbr\r!rPr`rrar)r[r+test_initializer_not_reuseds&   rc)F)rr)r`)N)F)rLrrYr~rr/rQrrrsrmathrr(rpicklerr tracebackr rrr r r Zjoblib._multiprocessing_helpersr Zjoblib.test.commonrrrrZjoblib.testingrrrrrZjoblib.externals.lokyrr rposix ImportErrorZ _openmp_test_helper.parallel_sumr distributedZjoblib._parallel_backendsrrrrrZjoblib.parallelr r!r"r#r$r%r&r'rZRETURN_GENERATOR_BACKENDSpoprkeysZALL_VALID_BACKENDSZPROCESS_BACKENDSZPARALLEL_BACKENDShasattrryr0r4r9r<r@ ExceptionrArPrSrUrYrdrergmarkrmrwrrrrrrrrrrrrrrrrrrrrr r!r&r*r+r-r0r1rr8r9r=r?rDrErFrHrMrPextendrbrQrRrVrXr[r\Z_parallel_backendsr]rdrgrkrorvrxrrrrrrrZ SQUARE_LOCALZ SQUARE_LAMBDArrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr rrrrZno_coverrr"r&r'r+r/r2r8r:r@rCrJrMrPrXr]r^rbrcr)r)r)r+s               ,             *               F (                                   (+                ," .  6        $     #