o \i1@s dZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl mZddlmZddlmZddlmZddlZddlZdd lmZmZmZdd lmZdd lmZmZmZmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%edurdd l&m'Z'ddl(m)Z)zddl*Z*Wn e+ydZ*Ynwzddl,m-Z-Wn e+ydZ-Ynwzddl.Z.Wn e+ydZ.Ynwddl/m0Z0m1Z1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7m8Z8m9Z9m:Z:mZm;Z;mZ?e?@dddgeAe6BZCeCdde6D7ZCedurgZDnddgZDeDdgZEeFedreCGeHdddZIddZJddZKd d!ZLGd"d#d#eMZNdTd%d&ZOd'd(ZPdUd)d*ZQd+d,ZRd-d.ZSd/d0ZTd1d2ZUe"d3egd?e"d@gdAdBdCZXe"d=eCe"d>ddDgdEdFZYe"d=eCdGdHZZdIdJZ[ee"dKgdLdMdNZ\ee"d=gdOdPdQZ]dRdSZ^e"dTe6e"dUe6dVdWZ_dXdYZ`edZd[Zad\d]Zbe"d>gd^d_d`Zce"d=eEdadbZdedcddZeeee"degdfe"dgddhgdidjZfe"d=eEdkdlZgee"d=eEdmdnZhee"d=eie?dohe"dpdqdrgdsdtZjee"d=eDdudvZkee"d=e6dwdxZldydzZme"d=e6e"d{dgd|fd}gd~fgddZnee"d=eEddZoddZpee"d=eDddZqddZrddZsddZtee"d=eEejVjuddddZvGddde3ZwddZxe"d=eCddZyee"d=eEe"d>gdddZzddZ{ddZ|e$eduddddZ}ddZ~eEddZeddedDee"d=ee"d3e;egde"d=eEe"d3egdddZddZddZejV"d>dDd}gee"d=eie?dohddZe"d=eCe"d>gdddZddZddZee"d=e?e"dpdqdrge"d>gdddZe"d=e?e"dpdqdrge"d>gdddZe"d=e?e"dpdqdrge"d>gdddZe"d=e?e"ddqdrge"ddqdrge"d>gdd d Ze"d gd e"ddqdrge"ddqdrgd dZeee"d=eDddZe"d=edurBddgngdddZee"d=eDddZee"d3eddDge"dgdddZe"d3egd:d;d<Zeee"d=gd>e"d>dDd6ge"d3edDd6ge"dAgdBe"d3egdEdFdGZ̐dHdIZ͐dTdJdKZee"d>dDd}ge"d=eDe"d3edDd}ge"d=eDdNdOZeejV"d>dDd}gdPdQZeejV"d>dDd}gdRdSZdS(Xz Test the parallel module. N) nullcontextsqrt TimeoutError) PicklingError)sleep)format_exception)dumploadparallel)mp)IS_GIL_DISABLEDnpwith_multiprocessing with_numpy)check_subprocess_call parametrizeraisesskipifwarns)get_reusable_executor)Queue) parallel_sum) LokyBackendMultiprocessingBackendParallelBackendBaseSequentialBackendThreadingBackend) BACKENDSParallel cpu_countdelayedeffective_n_jobsr parallel_backendparallel_configregister_parallel_backendmultiprocessingcCsg|]}t|qS)r ).0 backend_strr)r)z/var/www/www-root/data/www/176.119.141.140/sports-predictor/venv/lib/python3.10/site-packages/joblib/test/test_parallel.py Rsr-loky threading get_contextspawncCs ttjSN)r r DEFAULT_BACKENDr)r)r)r,get_default_backend_instance^s r4cCst|dt|ddS)N_pool_workers)getattrbackendr)r)r, get_workersfsr:cCs||Sr2r))xyr)r)r,divisionjr=cC|dSNr)r;r)r)r,squarenr>rCc@seZdZdZddZdS)MyExceptionWithFinickyInitz,An exception class with non trivial __init__cCdSr2r))selfabcdr)r)r,__init__uz#MyExceptionWithFinickyInit.__init__N__name__ __module__ __qualname____doc__rKr)r)r)r,rDr rDFcCs"|dkr|r tddddt|S)NrGrHrIrJ)rD ValueError)r;custom_exceptionr)r)r,exception_raiserysrVcCstdt)Ng?)timerKeyboardInterruptrBr)r)r,interrupt_raisers rYcCs|d||S)zNA module-level function so that it can be spawn with multiprocessing. rAr))r;r<zr)r)r,fsr[cCsttdSNr)typer get_active_backendr)r)r)r,_active_backend_typer_cCst||dddtdDS)Nn_jobsr9cs|] }tt|VqdSr2r#rCr*ir)r)r,   z parallel_func..r!range) inner_n_jobsr9r)r)r, parallel_funcsrmcCtdksJdSr\)r"r)r)r)r,test_cpu_countrocCrnr\r$r)r)r)r,test_effective_n_jobsrprrcontextzbackend_n_jobs, expected_n_jobs)ririrbNr)z positive-intz negative-intNone)idscCsT|d|dtdd|ksJWdn1swYtdddks(JdS)Nr/rurrq)rsbackend_n_jobsexpected_n_jobsr)r)r,test_effective_n_jobs_Nones r{r9rb)rrArtverbose)rA dcCs:ddtdDt|||dddtdDksJdS)NcSg|]}t|qSr)rCr*r;r)r)r,r-z(test_simple_parallel..rbr9r}csrcr2rdrr)r)r,rgz'test_simple_parallel..rkr!)r9rbr}r)r)r,test_simple_parallels rrAc Csd}td}Gdddt}|||dd}|ddt|Dt}|jD]}||}r=|\} } || | q't |d ksFJdS) Nrz(Done\s+\d+ out of \d+ \|)c@seZdZgZddZdS)z/test_parallel_pretty_print..ParallelLogcSs|j|dSr2)messagesappend)rFmsgr)r)r,_printr`z6test_parallel_pretty_print..ParallelLog._printN)rNrOrPrrr)r)r)r, ParallelLogrRri'rcSg|]}tt|qSr)r#r[rer)r)r,r-z.test_parallel_pretty_print..r) recompiler!rksetrsearchspanaddlen) r9rbn_taskspatternrexecutorlensmessagesrGrHr)r)r,test_parallel_pretty_prints   rcs|jtdddtjdd}td|ddd td D}|gd ks(JWdn1s2wYd d |D}|dvsGt|trRdfdd |D}t |dksZJdS)Nname!some_new_name_for_the_main_thread)targetrvalueTrecordrAracsrcr2rdrr)r)r,rgrhz6test_main_thread_renamed_no_warning..ri)rrcSs*g|]}dt|jvrt|jts|qS)zworker timeout)strr isinstanceDeprecationWarningr*wr)r)r,r-s z7test_main_thread_renamed_no_warning..)Nr(z3multi-threaded, use of fork() may lead to deadlockscsg|] }t|jvr|qSr))rrr message_partr)r,r-r) setattrr/current_threadwarningscatch_warningsr!rkrrr)r9 monkeypatchwarninforesultsr)rr,#test_main_thread_renamed_no_warnings(  rcCstjdd}tdt||dWdn1swYdd|D}|rI|rGtdd|D}tr=t|d knt|d k}|oF|Sd S|rMJdS) NTralways)r9rlcSsg|]}|jqSr))rrr)r)r,r-sz*_assert_warning_nested..css|] }d|jdvVqdS)zbacked parallel loops cannotrN)args)r*eachr)r)r,rg  z)_assert_warning_nested..rF)rr simplefilterrmallrr)r9rlexpectedrwarnings_are_correctwarnings_have_the_right_lengthr)r)r,_assert_warning_nesteds  rz%parent_backend,child_backend,expected))r.r(T)r.r.F)r(r(T)r(r.T)r/r(T)r/r.Tcsntd|dfddtdDtd|dfddtdD}|dkr/t|s-JdSt|s5JdS)NrArac3s"|] }ttdddVqdS)rFr9rlrNr#rr*_ child_backendr)r,rg( z0test_nested_parallel_warnings..rc3s"|] }ttddVqdS)rArNrrrrr)r,rg0rr/)r!rkanyr)parent_backendrrresr)rr,test_nested_parallel_warningss r)r.r(r/cCs>dg}dd}tj||fd}|||dsJdS)NFcSshtjdd}tddddtdDWdn1swYtt|t|dk|d<dS) NTrrArucs|] }ttdVqdS)皙?Nr#rrr)r)r,rgFrzPtest_background_thread_parallelism..background_thread..rr)rrr!rkprintr)is_run_parallelrr)r)r,background_threadDs  z=test_background_thread_parallelism..background_thread)rrr)r/Threadstartjoin)r9rrtr)r)r,"test_background_thread_parallelism?s rcCs"td|dddtdDdS)NrAracsr{Gz?Nrdrr)r)r,rgQrznested_loop..rjr8r)r)r, nested_loopPs"rrrcs&td|dfddtdDdS)NrArac3|] }ttVqdSr2)r#rrrr)r,rgWrhz#test_nested_loop..rj)rrr)rr,test_nested_loopTs rcCstr2)rTr8r)r)r,raise_exception\rLrc Cstt5tddd}|ttdttdgWdn1s$wYWddSWddS1s.N)rr!rkfullr)r)rr,test_mutate_input_with_threadsgs "r)rrAricCs6td}dd|Dt|ddd|DksJdS)z.Check the keyword argument processing of pmap. cSg|]}t|ddqSrr<r[rr)r)r,r-rrz(test_parallel_kwargs..rucs |] }tt|ddVqdSrrNrrr)r)r,rgr z'test_parallel_kwargs..Nr)rblstr)r)r,test_parallel_kwargsnsrcCstd}dd|D}td|d4}|j}||dd|Dks"J||dd|Dks/Jtdur>t|t|jus>JWdn1sHwYtdurZt|jdusZJ||d d|DksgJtdurtt|jdusvJdSdS) NrcSrrrrr)r)r,r-zrz4test_parallel_as_context_manager..rracsrrrrr)r)r,rgz3test_parallel_as_context_manager..csrrrrr)r)r,rgrcsrrrrr)r)r,rgr)rkr!_backendr r:)r9rrpmanaged_backendr)r)r, test_parallel_as_context_managerws"rcsdGdddtttddtdddfdd td DWd d S1s+wYd S) z`Check that pmap captures the errors when it is passed an object that cannot be pickled. c@seZdZddZdS)z1test_parallel_pickling..UnpicklableObjectcStd)N123) RuntimeErrorrFr)r)r, __reduce__r>z.UnpicklableObject.__reduce__N)rNrOrPrr)r)r)r,UnpicklableObjects rzthe task to sendmatchrAr.rac3s|] }ttVqdSr2r#idrrr)r,rgrz)test_parallel_pickling..rN)objectrrr!rkr)r)rr,test_parallel_picklings  "r byteorder)<>= max_nbytes1Mcsddtdd|djj}tdd|dfd d td D}|D]\}}||ks5J||jjks=Jtj |q+dS) NcSs ||jjfSr2)dtyperrBr)r)r,inspect_byteorder z=test_parallel_byteorder_corruption..inspect_byteorder)rArii4rAr.)rbr9rc3|] }tVqdSr2r#rr r;r)r,rgrhz5test_parallel_byteorder_corruption..ri) rarangereshapeviewrrr!rktestingassert_array_equal)rrinitial_np_byteorderresult x_returnedbyteorder_in_workerr)rr,"test_parallel_byteorder_corruptions  rcCs0ttd|ddddtdDdksJdS)NrArbr9timeoutcsr)gMbP?Nrrr)r)r,rgrhz0test_parallel_timeout_success..r)rr!rkr8r)r)r,test_parallel_timeout_successs rcCsNtttd|ddddtdDWddS1s wYdS)NrArrcsrrNrrr)r)r,rgrhz-test_parallel_timeout_fail..r)rrr!rkr8r)r)r,test_parallel_timeout_fails  "r sequential return_as generatorgenerator_unorderedcCsxttttd||ddddtdDWdn1s"wYttd||ddddtdDdS)NrAr)rbr9r!rcsrrrrr)r)r,rgrhz.rcsrrrrr)r)r,rgrh)rrlistr!rkr9r!r)r)r,)test_parallel_timeout_fail_with_generators r&c Cstdurtttd|dddtddDWdn1s$wYtttd|ddddDWdn1sEwYtd|d}t|jdusZJt|j}tt|ddtddDWdn1szwYt|jdusJt|j|usJd dtd D|d d td DksJt|j}tt|d ddDWdn1swYt|jdusJt|j|usJddtd D|dd td DksJ|j |j |j |j fWdn 1swYt|jdusJn!tttdddddDWdn 1s2wYtttddddtddDWdn 1sVwYtt tddddd tdDWddS1s{wYdS)NrAracSg|] \}}tt||qSr)r#r=r*r;r<r)r)r,r-rz&test_error_capture..)rr)rrcSrr)r#rYrr)r)r,r-rcSr'r)r(r)r)r)r,r-rcSrrrrr)r)r,r-rrcsrrrrr)r)r,rgrz%test_error_capture..cSrr)r*rr)r)r,r- rcSrrrrr)r)r,r-rcsrrrrr)r)r,rgrrucSrr)r*rr)r)r,r-"rcSr'r)r(r)r)r)r,r-'rr)rbr}csr)T)rUNr#rVrer)r)r,rg+rr)r rZeroDivisionErrorr!ziprXr:rrk _iteratingn_completed_tasksn_dispatched_tasks _abortingrD)r9r original_workersr)r)r,test_error_capturesd           '     $r3c Csddd}td|dw}ttdd|dd |dd DWdn1s)wYttdd|d d |d d DWdn1sKwYttdd|d d |dd DWdn1smwYWddSWddS1swYdS)Nrcss*tdD] }||krtd|VqdS)NIterator Raising Error)rkrT)raise_atrfr)r)r, my_generator2s  z1test_error_in_task_iterator..my_generatorrArar5rcsrcr2rdrer)r)r,rg;rz.test_error_in_task_iterator..)r6csrcr2rdrer)r)r,rg@rrcsrcr2rdrer)r)r,rgDrr)r!rrT)r9r7rr)r)r,test_error_in_task_iterator/s  "r:cCs|d|dS)Nz Consumed %s)r)queueitemr)r)r,consumerGrpr=zbatch_size, expected_queue) Produced 0 Consumed 0 Produced 1 Consumed 1 Produced 2 Consumed 2 Produced 3 Consumed 3 Produced 4 Consumed 4 Produced 5 Consumed 5r) r>r@rBrDr?rArCrErFrHrGrIcsTtfdd}td||dfdd|D|ks Jtdks(JdS) z=Test that with only one job, Parallel does act as a iterator.c3(tdD] }d||VqdSNr z Produced %irkrrfr;r)r,producerx  z'test_dispatch_one_job..producerrrb batch_sizer9c3s|] }tt|VqdSr2r#r=rrNr)r,rg}rz(test_dispatch_one_job.. N)r$r!r)r9rRexpected_queuerOr)rNr,test_dispatch_one_jobKs+  rVcst}|fdd}tddd|dfdd|Dt}|d d ks,J|d d d }|dks;J|d}||ksFJtdksNJd S)zRCheck that using pre_dispatch Parallel does indeed dispatch items lazily. c3rJrKrLrMrNr)r,rOrPz/test_dispatch_multiprocessing..producerrArri)rbrR pre_dispatchr9c3s|] }ttdVqdS)rNrSrrNr)r,rgrz0test_dispatch_multiprocessing..rr>Nrz Consumed anyrtrDrT)r Managerr$r!indexr)r9managerrOqueue_contentsfirst_consumption_indexproduced_3_indexr)rNr,test_dispatch_multiprocessings    r^cCs\tdddd}|ddtdD|jdksJWddS1s'wYdS) NrAautor/rQcsrcr2rrer)r)r,rgrz/test_batching_auto_threading..rr!rkrcompute_batch_size)rr)r)r,test_batching_auto_threadings"rccCs\tdd|d}|ddtdD|jdksJWddS1s'wYdS)NrAr_rQcsrcr2rrer)r)r,rgrz2test_batching_auto_subprocesses..r`rra)r9rr)r)r,test_batching_auto_subprocessess"rdcCsNtttddddddtdDWddS1s wYdS) zCMake sure that exception raised during dispatch are indeed capturedrAr)rbrWr}csrcr2r+rer)r)r,rgrhz*test_exception_dispatch..rN)rrTr!rkr)r)r)r,test_exception_dispatchs  "rfcC tddddtdDdS)NrArucsrcr2r+r*jr)r)r,rgrz(nested_function_inner..rrjrMr)r)r,nested_function_inner rjcCrg)NrArucsrcr2)r#rjrhr)r)r,rgrz(nested_function_outer..rrjrMr)r)r,nested_function_outerrkrlz'https://github.com/joblib/loky/pull/255reasoncCstt}td|dddtdDWdn1swYt|j|j|j}d|}d|vs7Jd |vs=Jd |vsCJt|jtusLJdS) zEnsure errors for nested joblib cases gets propagated We rely on the Python 3 built-in __cause__ system that already report this kind of information to the user. rAracsrcr2)r#rlrer)r)r,rgrhz1test_nested_exception_dispatch..rNrlrjrV) rrTr!rkr r]rtbr)r9excinfo report_linesreportr)r)r,test_nested_exception_dispatchs      rtc@s$eZdZdZdddZd ddZdS) FakeParallelBackendz8Pretends to run concurrently while running sequentially.rNcKs|||_||_|Sr2)r$rbr )rFrbr backend_argsr)r)r, configures zFakeParallelBackend.configurecCs"|dkrttd|d}|S)Nrr)maxr r"rFrbr)r)r,r$sz$FakeParallelBackend.effective_n_jobsr)r)rNrOrPrQrwr$r)r)r)r,rus ruc Csttdd tddWdn1swYttddtddWdn1s2wYWdn1sAwYttdd&tddWdn1s]wYWddSWddS1suwYdS)NzInvalid backend:rz unit-testingr8)rrTr!r&r)r)r)r,test_invalid_backends    "rzcCs,tt}td|dWdn1swYdt|jvs%Jtt}td|dWdn1s.rcss |] \}}|t|kVqdSr2r)r*rfrr)r)r,rgr)r!_effective_n_jobsrkr enumerate)r9rbrrr)r)r,test_njobs_converted_to_ints rcCs<ztdtdtvs JtdtksJWtd=dStd=w)N test_backend)r'rur r)r)r)r,test_register_parallel_backends   rcCs^tj}ttks JztdtdddttksJW|t_n|t_wttks-JdS)Nr/T) make_default)r r3r_r4r'r r)default_backend_origr)r)r,test_overwrite_default_backend%srzOnly without multiprocessingcCsttddtddddtdDWdn1swYtddtddtdDWddS1sAwYdS) Nz)joblib backend '.*' is not available on.*rr.r8csrcr2rdrer)r)r,rg4rz2test_backend_no_multiprocessing..ricsrcr2rdrer)r)r,rg8r)r UserWarningr!rkr&r)r)r)r,test_backend_no_multiprocessing1s  "rcCs^||ddt\}}|dksJtddksJt}|jdks%J|dkr;t|tus1Jt|jtus:JnB|dkrQt|tusGJt|jtusPJn4|dkrgt|t us]Jt|jt usfJn.| drt|t ustJt|jt usJWddSWddSWddSWddSWddS1swYdS)Nrirur(r.r/test_) r r^r$r!rbr]rrrr startswithru)rs backend_nameactive_backend active_n_jobsrr)r)r,check_backend_context_manager;s8    "rcCsg|]}d|qS)ztest_backend_%dr)rer)r)r,r-Qrricsr|tvr |t|tttksJt|ttks JtdddfddtDttks7JdS)NrAr/rac3s"|] }|stt|VqdSr2)r#r)r*rHrsr)r,rgcs  z/test_backend_context_manager..)r setitemrur_r4rr! all_backends_for_context_manager)rr9rsr)rr,test_backend_context_managerTs rc@seZdZdZdddZdS)ParameterizedParallelBackendz9Pretends to run conncurrently while running sequentially.NcCs|durtd||_dS)Nzparam should not be None)rTparam)rFrr)r)r,rKps z%ParameterizedParallelBackend.__init__r2rMr)r)r)r,rmsrcCs|tdtttksJ|dddd?t\}}t|tus%J|jdks,J|dks2Jt }|j dks.rcSrr)rrer)r)r,r-rz>test_parameterized_backend_context_manager..) rr rr_r4r r^r]rr!rbrrk)rrsrrrrr)r)r,*test_parameterized_backend_context_managervs   rcCsttksJ|tdddd?t\}}t|tus J|jdks'J|dks-Jt}|jdks7J|j |us>J|ddt dD}Wdn1sSwY|ddt dDkseJttksmJdS) N+)rrrucsrcr2rrer)r)r,rgrzFtest_directly_parameterized_backend_context_manager..cSrr)rrer)r)r,r-rzGtest_directly_parameterized_backend_context_manager..) r_r4rr r^r]rr!rbrrk)rsrrrrr)r)r,3test_directly_parameterized_backend_context_managers   rcCstdtS)Nr)rosgetpidr)r)r)r,sleep_and_return_pidsrcCs<ttksJtdksJtddddtdDS)NrrArucs|]}ttVqdSr2)r#rrr)r)r,rgz"get_nested_pids..)r_rr!rrkr)r)r)r,get_nested_pidssrcs eZdZdZfddZZS) MyBackendz:Backend to test backward compatibility with older backendscstt|dSr\)superrget_nested_backendr __class__r)r,rszMyBackend.get_nested_backend)rNrOrPrQr __classcell__r)r)rr,rsrback_compat_backend)r/r.r(rcCsh||&tddddtdD}|D] }tt|dks!JqWddS1s-wYdS)NrArucsrr2)r#rrr)r)r,rgrz6test_nested_backend_context_manager..rr)r!rkrr)rsr9 pid_groups pid_groupr)r)r,#test_nested_backend_context_managers "r)rArtNcs~ddtddfddtdD|dtddfddtdDWddS1s8wYdS) NcSs0tt|ks Jt|}t|ksJdSr2)r_r r$r!r)expected_backend_typeexpected_n_jobr)r)r,check_nested_backendsz?test_nested_backend_in_sequential..check_nested_backendrruc3s |] }ttjdVqdSr)r#r r3r)rr)r,rgrz4test_nested_backend_in_sequential..rc3s|] }tVqdSr2rrr9rrbr)r,rgrrj)r9rbrsr)rr,!test_nested_backend_in_sequentials  "rcCs\|| }|tur |d}|tur|d}|j|ksJWddS1s'wYdS)Nr9r)r&r% nesting_level)rs inner_backendexpected_levelctxr9r)r)r,check_nesting_levels "r outer_backendrcst|dtd|dfddtdDddtfddtdDWddS1s9wYdS) NrrArac3 |] }ttdVqdSrr#rrrsrr)r,rgrz-test_backend_nesting_level..rruc3rrrrrr)r,rgr)rr!rk)rsrrr)rr,test_backend_nesting_levels  "rwith_retrieve_callbackTcsddlGfdddt}td|dd|d;}tddfd d td D|tur:|d jd ks:J|turG|djd ksOJWddSWddS1sZwYdS)Nrcs"eZdZdZZjddZdS)z)test_retrieval_context..MyBackendrcss|jd7_dVdSrvrMrr)r)r,retrieval_context s z;test_retrieval_context..MyBackend.retrieval_contextN)rNrOrPrfsupports_retrieve_callbackcontextmanagerrr)) contextlibrr)r,rs r retrievalcSstddddt|DS)NrArucsrcr2rrer)r)r,rgrz>test_retrieval_context..nested_call..rj)nr)r)r, nested_callsz+test_retrieval_context..nested_callrAruc3s|] }t|VqdSr2rre)rr)r,rgrz)test_retrieval_context..rr9r)rrr'r!rkr&rfr%)rsrrrr))rrrr,test_retrieval_contexts   "rrR)rrtgQ?cCs8ttt|dWddS1swYdS)NrR)rrTr!rr)r)r,test_invalid_batch_sizes  "rz)n_tasks, n_jobs, pre_dispatch, batch_size) )rArArr_)rArArbr_rrArbr_)irArbr_r)rrrbr_)rTrbr_)rT 2 * n_jobsr)rTrr)rrTrrS)rrTrr_cCsN|||d}ddt|D}tdi|ddt|D}||ks%JdS)N)rbrWrRcSrr)rrer)r)r,r-:rz0test_dispatch_race_condition..csrcr2rdrer)r)r,rg;rz/test_dispatch_race_condition..r)r)rrbrWrRparamsrrr)r)r,test_dispatch_race_condition%s  rcCs8t}tddd}|jd}|}||ksJdS)NrAr(rars)r get_start_methodr!_backend_kwargsget)mp_start_methodrrs start_methodr)r)r,test_default_mp_context?s   rcs^|dkr td}tjd}|ddtjtd|dfddt dDdS) Nr(r1rirArac3s"|] }ttjjVqdSr2)r#rdotTrerGr)r,rg^s zAtest_no_blas_crash_or_freeze_with_subprocesses..) r r0rrandom RandomStaterandnrrr!rk)r9rngr)rr,.test_no_blas_crash_or_freeze_with_subprocessesHs    &rafrom joblib import Parallel, delayed def square(x): return x ** 2 backend = "{}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) cCsB|dkrtdkrtdt|}ttjd|gddddS)Nr(forkzVRequire fork start method to use interactively defined functions with multiprocessing.z-cr\[0, 1, 4, 9, 16\]r stdout_regex) r rpytestskip,UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_NO_MAINformatrsys executable)r9coder)r)r,2test_parallel_with_interactively_defined_functionsqs  raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed def run(f, x): return f(x) {define_func} if __name__ == "__main__": backend = "{backend}" if backend == "spawn": from multiprocessing import get_context backend = get_context(backend) callable_position = "{callable_position}" if callable_position == "delayed": print(Parallel(n_jobs=2, backend=backend)( delayed(square)(i) for i in range(5))) elif callable_position == "args": print(Parallel(n_jobs=2, backend=backend)( delayed(run)(square, i) for i in range(5))) else: print(Parallel(n_jobs=2, backend=backend)( delayed(run)(f=square, x=i) for i in range(5))) z!def square(x): return x ** 2 zcdef gen_square(): def square(x): return x ** 2 return square square = gen_square() zsquare = lambda x: x ** 2 define_funccallable_position)r#rkwargsc Csv|dvr|tks tjdkrtdtj|||tj tj t j d}| d}| |ttj|jgddddS) N)r(r1win32zNot picklable with pickle)rr9rjoblib_root_folderzunpicklable_func_script.pyrrr) SQUARE_MAINrplatformrr)UNPICKLABLE_CALLABLE_SCRIPT_TEMPLATE_MAINrrpathdirnamejoblib__file__rwriterrstrpath)r9rrtmpdirr code_filer)r)r,0test_parallel_with_unpicklable_functions_in_argss      raimport sys import faulthandler # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed from functools import partial class MyClass: '''Class defined in the __main__ namespace''' def __init__(self, value): self.value = value def square(x, ignored=None, ignored2=None): '''Function defined in the __main__ namespace''' return x.value ** 2 square2 = partial(square, ignored2='something') # Here, we do not need the `if __name__ == "__main__":` safeguard when # using the default `loky` backend (even on Windows). # To make debugging easier faulthandler.dump_traceback_later(30, exit=True) # The following baroque function call is meant to check that joblib # introspection rightfully uses cloudpickle instead of the (faster) pickle # module of the standard library when necessary. In particular cloudpickle is # necessary for functions and instances of classes interactively defined in the # __main__ module. print(Parallel(backend="loky", n_jobs=2)( delayed(square2)(MyClass(i), ignored=[dict(a=MyClass(1))]) for i in range(5) )) )rcCs.|d}|tttj|jgddddS)Nz(joblib_interactively_defined_function.pyr)rr)rr5INTERACTIVE_DEFINED_FUNCTION_AND_CLASS_SCRIPT_CONTENTrrrrrscriptr)r)r,7test_parallel_with_interactively_defined_functions_lokys    raimport sys # Make sure that joblib is importable in the subprocess launching this # script. This is needed in case we run the tests from the joblib root # folder without having installed joblib sys.path.insert(0, {joblib_root_folder!r}) from joblib import Parallel, delayed, hash import multiprocessing as mp mp.util.log_to_stderr(5) class MyList(list): '''MyList is interactively defined by MyList.append is a built-in''' def __hash__(self): # XXX: workaround limitation in cloudpickle return hash(self).__hash__() l = MyList() print(Parallel(backend="loky", n_jobs=2)( delayed(l.append)(i) for i in range(3) )) cCs0|d}|tttj|jgdddddS)Nz)joblib_interactive_bound_method_script.pyz\[None, None, None\] LokyProcess)r stderr_regexr)rr9INTERACTIVELY_DEFINED_SUBCLASS_WITH_METHOD_SCRIPT_CONTENTrrrrrr)r)r,:test_parallel_with_interactively_defined_bound_method_lokys    rcCs"tg}tdd|gksJdS)NrAru)iterr!)exhausted_iteratorr)r)r,%test_parallel_with_exhausted_iterator)srcCs"t|tjs tdt||S)Nz#Expected np.memmap instance, got %r)rrmemmap TypeErrorr]copyrr)r)r, check_memmap.s rcCsdd}tdd|ddd|dD}t||t|D] \}}tj||qtd d|dd d|dD}t||t|D] \}}tj||qBdS) Ncss*t|D] }tjdtjd|VqdS)Nrr)rkronesfloat32)rrfr)r)r,generate_arrays<s zBtest_auto_memmap_on_arrays_from_generator..generate_arraysrArrbrr9csrcr2r#rr*rGr)r)r,rgBrhz.rrcsrcr2rrr)r)r,rgKrh)r!r-rrrr)r9r rrrr)r)r,)test_auto_memmap_on_arrays_from_generator4srcCs|Sr2r))argr)r)r,identityRrLrcs|dj}tj}tj|ddtj|ddg}t||t|ddt ddfdd d D\}t d tj s;Jd j |ksDJtj ||dS) Nz test.mmapuint8r r) mmap_moderAruc3rr2)r#rrrr)r,rg^rz.test_memmap_with_big_offset..r9r)rrmmapALLOCATIONGRANULARITYrzerosr r r r!rroffsetrr)rfnamesizeobjrr)rr,test_memmap_with_big_offsetVs   rcCstjdd}tdddddtdDWdn1s wYt|dks-J|d}t|jts9Jt|jd ksBJdS) NTrr)rbrcsrcr2rdrer)r)r,rgfrzFtest_warning_about_timeout_not_supported_by_backend..2rzThe backend class 'SequentialBackend' does not support timeout. You have set 'timeout=1' in Parallel but the 'timeout' parameter will not be used.) rrr!rkrrrrr)rrr)r)r,3test_warning_about_timeout_not_supported_by_backendds  rcCs |||<|Sr2r)) input_listrYrr)r)r,set_list_valueqsr!)rrArcsVdgdt|dddfddtdD}t|}tddt|Ds)JdS) Nrrr"r/rbr!r9c3s |] }tt||VqdSr2)r#r!rer r)r,rg|rzPtest_parallel_return_order_with_return_as_generator_parameter..cs|] \}}||kVqdSr2r)r*vrr)r)r,rgr)r!rkr$rr-)rbrr)r#r,=test_parallel_return_order_with_return_as_generator_parametervs  r'cCs|rtdt|S)Nr)rr)edelayr)r)r,_sqrt_with_delaysr*cspt|d|dddtdDtfddtdD}dgttd d}td dt||Ds5JdS) Nr#r"css&|]}tt|d|dkVqdS)rArN)r#r*rer)r)r,rgs zK_test_parallel_unordered_generator_returns_fastest_first..rc3s|]}tVqdSr2)nextrrr)r,rgs rrAcsr$r2r)r%r)r)r,rgr)r!rksortedr$rr-)r9rbquickly_returnedexpected_quickly_returnedr)r,r,8_test_parallel_unordered_generator_returns_fastest_firstsr1cCst||dSr2)r1r9rbr)r)r,7test_parallel_unordered_generator_returns_fastest_firstsr3)rrAr|rtcCsvdgdgd}ttt}t||ddd|DWdn1s(wYt|}|dks9JdS)NrGrrracss|] }ttj|VqdSr2r#rWrrer)r)r,rgrz%test_abort_backend..r4)rrrWr!)rbr9delayst_startdtr)r)r,test_abort_backends  r8cCstjtdtd}d|d<|S)NgAr Fr)rr intbool)rrr)r)r,get_large_objectsr;cCs\t|||d}|ddtdD}t|t|~WddS1s'wYdS)N)rbr9r!csrcr2)r#r;rer)r)r,rgrz0_test_deadlock_with_generator..r)r!rkr+)r9r!rbr rr)r)r,_test_deadlock_with_generators "r<cCst|||dSr2)r<)r9r!rbr)r)r,test_deadlock_with_generatorsr=cCsttdd)t|||d}|ddtdD}t}|ddtdD}Wdn1s2wYt|d ksCJd ~dS) N)This Parallel instance is already runningrr%csrrrrr)r)r,rgrz/test_multiple_generator_call..rcsrcr2rrer)r)r,rgrrrA[The error should be raised immediately when submitting a new task but it took more than 2s.)rrr!rkrW)r9r!rbr gr6gen2r)r)r,test_multiple_generator_callsrBc Cst|||dF}|ddtdD}t}ttdd|ddtdD}Wdn1s3wYt|d ksDJd Wd~dS1sPwY~dS) Nr%csrrrrr)r)r,rgrz7test_multiple_generator_call_managed..rr>rcsrcr2rrer)r)r,rgrrrAr?)r!rkrWrr)r9r!rbr r@r6g2r)r)r,$test_multiple_generator_call_manageds   rD return_as_1 return_as_2cCst|||dddtdD}t|||dddtddD}|dkr)t|}|dkr1t|}tddt|tdDsAJtd dt|tddDsRJdS) Nr%cs |] }tt|dVqdSrANrrer)r)r,rgrz9test_multiple_generator_call_separated..rcsrGrHrrer)r)r,rgrr4r#csr$r2r)r*rrfr)r)r,rgrcsr$r2r)rIr)r)r,rgr)r!rkr.rr-)r9rErFrbr@rCr)r)r,&test_multiple_generator_call_separateds &rJzbackend, error))r.T)r/F)r Fc Cs\|dkr tdur tdtd||d}|ddtdD}t|dd }|r/ttd d nt }|6t }td||dd dtdd D}|dkrRt |}t ddt |tdd DscJWdn1smwYt |dks|Jd} |jr| dkr| d7} t d|jr| dkst |dksJ|dkr|jsJdSdS)Nr.zRequires multiprocessingrAr%csrrrrer)r)r,rg&rz.rcSstdS)NzGenerator collected)rr)r)r)r,'z;test_multiple_generator_call_separated_gc..z The executor underlying ParallelrcsrGrHrrer)r)r,rg3rr4r#csr$r2r)rIr)r)r,rg:rrrrirr{)r rrr!rkweakreffinalizerrrrWr.rr-aliverrr1) r9rErFerrorr r@g_wrrr6retryr)r)r,)test_multiple_generator_call_separated_gcs: $   rScCs|j}tdd||d#}|ddtjdgdDtt|dks'JWdn1s1wYtdD] }t|sCn td q:t d tdd|d }|d dtjdgdDtdD]}t|sqdStd qgt d ) NrAr)rbrr9 temp_foldercsrcr2rrr)r)r,rgVrz(test_memmapping_leaks..rrrrz/temporary directory of Parallel was not removedr csrcr2rrr)r)r,rgfr) rr!rrrrlistdirrkrAssertionError)r9rrrr)r)r,test_memmapping_leaksKs"        rW)Nr.r/cCs<td|dddtdD}|ddtdDksJdS)NrAracss |] }tdd|VqdS)cSr?r@r)rBr)r)r,rKvrLz2test_lambda_expression...Nrrer)r)r,rgurz)test_lambda_expression..rcSg|]}|dqSrAr)rer)r)r,r-xrz*test_lambda_expression..rj)r9rr)r)r,test_lambda_expressionpsrZcsd}d}d|td||d}|fddt|D|jj|jjks&J|jj|jjks0J|fddt|D|jj|jjksGJ|jj|jjksQJd S) zCTest that a parallel backend correctly resets its batch statistics.rAig@r)r}rbr9c3|] }ttjVqdSr2r4re task_timer)r,rgrz6test_backend_batch_statistics_reset..c3r[r2r4rer\r)r,rgrN)r!rkr_effective_batch_size_DEFAULT_EFFECTIVE_BATCH_SIZE_smoothed_batch_duration _DEFAULT_SMOOTHED_BATCH_DURATION)r9rbn_inputsrr)r\r,#test_backend_batch_statistics_reset{s rccCsdD]<}tt|djtusJt|dd}t|jtus Jt|dd}t|jtus/Jt|dd}t|jtus>Jqtdd dd }t|jtusOJ|d ddtdd }t|jtusdJ|jdkskJWdn1suwY|d ddtd dd}t|jtusJ|jd ksJWdn1swY|d ddtdd }t|jtusJ|jdksJWdn1swY|d ddtd dd}t|jtusJ|jd ksJWddS1swYdS)N)rrArtruthreadsrbprefer processes sharedmemrbrequirerAr.)rbr9rfrfrirjr)r]r!rr4rrrb)rsrbrr)r)r,$test_backend_hinting_and_constraintss<       "rmrf)NrgrdcCsddtdD}t||dddtdD}||ksJt||dtddtdD}Wdn1s;wY||ksFJdS) NcSrXrYr)rer)r)r,r-rz7test_backend_hinting_always_running..rrecsrcr2rdrer)r)r,rgrhz6test_backend_hinting_always_running..)rfrbcsrcr2rdrer)r)r,rgr)rkr!r&)rbrfexpected_resultsrr)r)r,#test_backend_hinting_always_runnings rocCszGdddt}||&tddd}t|j|usJtddd}t|j|us,JWdn1s6wYGdd d t}||Otddd}t|j|usXJ|\}}|d ksdJ|d ksjJtddd d }t|jtuszJ|\}}d }||ksJ|d ksJWdn1swYttt|ddWddS1swYdS)Nc@$eZdZdZdZddZddZdS)z[test_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackendTcSrEr2r)rr)r)r, apply_asyncrLzgtest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.apply_asynccS|Sr2r)ryr)r)r,r$rLzltest_backend_hinting_and_constraints_with_custom_backends..MyCustomThreadingBackend.effective_n_jobsNrNrOrPsupports_sharedmem use_threadsrqr$r)r)r)r,MyCustomThreadingBackend  rvrArgrerhric@rp)z\test_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackendFcSrEr2r)rr)r)r,rqrLzhtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.apply_asynccSrrr2r)ryr)r)r,r$rLzmtest_backend_hinting_and_constraints_with_custom_backends..MyCustomProcessingBackend.effective_n_jobsNrsr)r)r)r,MyCustomProcessingBackendrwrxror)rbrjr}zUsing ThreadingBackend as joblib backend instead of MyCustomProcessingBackend as the latter does not provide shared memory semantics.r9rj) rr!r]r readouterrrstriprrT)capsysrsrvrrxouterrrr)r)r,9test_backend_hinting_and_constraints_with_custom_backendss2        "rcCstt tddWdn1swYtt tddWdn1s-wYtttdddWdn1sGwYtdurtttdddWdn1sewYtttd ddWddS1swYdSdS) Ninvalidrkrlrgrh)rfrjr.ryr()rrTr!r r)r)r)r,,test_invalid_backend_hinting_and_constraints s"       "rc stdd4}t|jj|jjfg}dkr|WdS|fddtdD}||dWdS1s.r)r!r]rrNrrk)rrr this_levelrr)rr,r s  $rcCs^||dd t}Wdn1swY|d}|dfdddg}||ks-JdS)NrAruBackendr)rr)rrA)rtitle)rsr9backend_types_and_levelstop_level_backend_typeexpected_types_and_levelsr)r)r,test_nested_parallelism_limit.s rcCstddtdDS)z6A horrible function that does recursive parallel callscsrr2)r#_recursive_parallelrer)r)r,rgArz&_recursive_parallel..rArj) nesting_limitr)r)r,r?src Cs||ddtt }tWdn1swYWdn1s(wY|j}|dkrFddlm}t||tfrFt dt|t sMJdS)NrArur.r)TerminatedWorkerErrorz1Loky worker crash when serializing RecursionError) r BaseExceptionrr&joblib.externals.loky.process_executorrrrrxfailRecursionError)rsr9rqexcrr)r)r,test_thread_bomb_mitigationDs   rcCs*i}dD] }tj|||<q|tdfS)N)OMP_NUM_THREADSOPENBLAS_NUM_THREADSMKL_NUM_THREADSVECLIB_MAXIMUM_THREADSNUMEXPR_NUM_THREADSNUMBA_NUM_THREADS ENABLE_IPCr)renvironrr)env_varsvarr)r)r,_run_parallel_sum`s rzNeed OpenMP helper compiledcCstd|dddtdD}ttdd}|D]-\}}||ks#J|D]\}}|dr9|t|ks8Jq'|dks?J|dksEJq'qdS) NrAracsrr2)r#rrr)r)r,rgr  z-test_parallel_thread_limit..r_THREADSr1)r!rkrxr"itemsendswithr)r9rexpected_num_threadsworker_env_varsomp_num_threadsrrr)r)r,test_parallel_thread_limitos    rzThis test requires daskcCs:ttdd |dWddS1swYdS)NzPlease install daskrdask)rrTrr)r)r,)test_dask_backend_when_dask_not_installeds "rc CsGdddt}d}||6tjt|dtddddtdDWdn1s0wYWddSWddS1sHwYdS) Nc@s&eZdZddZdddZddZdS) z3test_zero_worker_backend..ZeroWorkerBackendc_dSr\r))rFrrr)r)r,rwrLz=test_zero_worker_backend..ZeroWorkerBackend.configureNcSr)NzNo worker availabler)rFfunccallbackr)r)r,rqr>z?test_zero_worker_backend..ZeroWorkerBackend.apply_asynccSrr\r)ryr)r)r,r$rLzDtest_zero_worker_backend..ZeroWorkerBackend.effective_n_jobsr2)rNrOrPrwrqr$r)r)r)r,ZeroWorkerBackends  rz&ZeroWorkerBackend has no active workerrrArucsrcr2rrer)r)r,rgrz+test_zero_worker_backend..)rrrrr!rk)rsr expected_msgr)r)r,test_zero_worker_backends "rcsdadddks JtddfddtdD}t|dhks&Jdadks/Jtddfd dtdD}t|dhksHJdS) Nzoriginal valuecSstSr2)MY_GLOBAL_VARIABLEr)r)r)r, check_globalssz@test_globals_update_at_each_parallel_call..check_globalsrAruc3|]}tVqdSr2rrerr)r,rgrz.z changed valuec3rr2rrerr)r,rgr)rr!rkr)workers_global_variabler)rr,)test_globals_update_at_each_parallel_callsrcCs4ddl}|jdd}|||td}|S)Nrr threadpoolctl)numpyrrrr importorskipthreadpool_info)rrGrr)r)r,_check_numpy_threadpool_limitss   rcCs4|D]}|d|dkr|dSqtd|)Nfilepath num_threadsz,An unexpected module was loaded in child: {})rTr) child_module parent_info parent_moduler)r)r,_parent_max_num_threads_fors rcCs@|D]}|D]}t||}t|||h}|d|vsJqqdS)Nr)rmin) workers_inforrchild_threadpool_inforparent_max_num_threadsrr)r)r,check_child_num_threadssr)rArr|rtcCszt}t|dkrtjddtd|dddtdD}t|}|d kr-|dd }ntt|d }t |||dS) Nr&Need a version of numpy linked to BLASrmr.r2csrr2r#rrer)r)r,rgrz;test_threadpool_limitation_in_child_loky..rArr rrrrr!rkr$rxr"r)rbrworkers_threadpool_infosexpected_child_num_threadsr)r)r,(test_threadpool_limitation_in_child_lokys  rinner_max_num_threads)rrArNcCst}t|dkrtjdd|d|dt|dddtd D}Wdn1s.wYt|}|d krB|dd }n|durOtt|d }n|}t |||dS) Nrrrmr.rrucsrr2rrer)r)r,rg rz>test_threadpool_limitation_in_child_context..rArrr)rsrbrrrrr)r)r,+test_threadpool_limitation_in_child_contexts"   rvar_name)rrrc st|dkr tdtddddtj}z_dtj<t|dfd d t d D}|ddgks;J|d dd t|dfdd t d D}Wdn1s]wY|ddgksjJW|durutj=dS|tj<dS|durtj=w|tj<w)NrzSkip test when n_jobs == 1TreusecSs tj|Sr2)rrr)rr)r)r,_get_env,r z>test_threadpool_limitation_in_child_override.._get_env4ruc3r r2rrerrr)r,rg3rz?test_threadpool_limitation_in_child_override..rAr.rc3r r2rrerr)r,rg7rhr) r$rrrshutdownrrrr!rk)rsrbroriginal_var_valuerr)rr,,test_threadpool_limitation_in_child_overrides*   "  r)rArrtcCsJdd}||tdd}tdD]}||tdd}||ks"JqdS)Ncs,tdt|dfddtdDdS)Nrruc3rr2)r#sumrerBr)r,rgKrzAtest_loky_reuse_workers..parallel_call..rrur)rBr, parallel_callIs$z.test_loky_reuse_workers..parallel_callTrr)rrk)rbrfirst_executorrrr)r)r,test_loky_reuse_workersCs   rcCsd|t<dS)N initialized)rr)statusr)r)r,_set_initializedYr`rcCst}||d}|dvsJd||sdSd||<td}tdd|D}t|krR||krRtdtdd|D}t|krR||ks8t|kr\td |S) N)rstartedz2worker should have been in initialized state, got rrcSg|] \}}|dkr|qSrr)r*pidr&r)r)r,r-iz!_check_status..rcSrrr)rr)r)r,r-lrz-Waited more than 30s to start all the workers)rrrrWrrrr)rrb wait_workersrstatedeadline n_startedr)r)r, _check_status]s"     rcsft}|||tfdtfddtdDWddS1s,wYdS)Nr9rb initializerinitargsc3|] }ttVqdSr2r#rrerbrr)r,rgrz+test_initializer_context..r)r rXdictrr!rk)rbr9rsrZr)rr,test_initializer_contextts "rcs>t}|t|tfdfddtdDdS)Nrc3rr2rrerr)r,rgrz,test_initializer_parallel..r)r rXrr!rrk)rbr9rZr)rr,test_initializer_parallelsrcsxd}t}|t}t|D]}tdtfdfddtD}|t|}qt|ks:JddS)Nrir.rc3"|] }ttddVqdST)rNrrerr)r,rg  z*test_initializer_reused..z=The workers should be reused when the initializer is the same) r rXrrrkr!runionrrb n_repetitionsrZpidsrfrr)rr,test_initializer_reuseds&   rcs|d}t}t}t|D]"}|tdtfdfddtD}|t|}q t||ks.zFThe workers should not be reused when the initializer arguments change) r rXrrkrr!rrrrr)rr,test_initializer_not_reuseds&   r)F)rr)rir2)rQrrrrr/rWrrMrrmathrr(rpicklerr tracebackr rrr r r joblib._multiprocessing_helpersr joblib.test.commonrrrrjoblib.testingrrrrrjoblib.externals.lokyrr;rposix ImportError _openmp_test_helper.parallel_sumr distributedjoblib._parallel_backendsrrrrrjoblib.parallelr r!r"r#r$r%r&r'rRETURN_GENERATOR_BACKENDSpopr.keysALL_VALID_BACKENDSPROCESS_BACKENDSPARALLEL_BACKENDShasattrrr0r4r:r=rC ExceptionrDrVrYr[r_rmrorrmarkr{rrrrrrrrrrrrrrrrrrr&r3r:r=rVr^rcrdrfrjrlrrtrurzr~rrrrrrextendrkrrrrrr_parallel_backendsrrrrrrrrrrrrrr SQUARE_LOCAL SQUARE_LAMBDArrrrrrrrrrrrrrrr!r'r*r1r3r8r;r<r=rBrDrJrSrWrZrcrmrorrrrrno_coverrrrrrrrrrrrrrrrrrrrr)r)r)r,s              ,            *                F  (                              (+                    ,"  .  6        $      #