
    3[g"                     |   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	Z	d dl
Z
d dlZd dlZd dlmZmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dl m!Z! d dl m"Z" d dl m#Z# d dl m$Z$ d dl m%Z% d dl m&Z& d dl m'Z( d Z)d Z*d Z+d Z,d Z-eed               Z.e eej                  dk7  xs dd      d               Z/eed               Z0ed        Z1ee edeejd                  gdd g!      d"                      Z3ee edeejd                  gdd g!      d#                      Z4ee ed$dd g      d%                      Z5ee ed$dd g      d&                      Z6ee ed$dd g      d'                      Z7ee ed$dd g      d(                      Z8eed)               Z9eed*               Z:ee ed$dd g      d+                      Z;ee ed$dd g      d,                      Z<ee ed$dd g      d-                      Z=ee edeejd                  gdd g!      d.                      Z>ee ed$ ej~                  dej                  j                  d/      0      d g      d1                      ZBee edeejd                  gdd g!      d2                      ZCeee edeejd                  gdd g!      d3                             ZDeee edeejd                  gdd g!      d4                             ZEee edeejd                  gdd g!      d5                      ZFd6 ZGee edeejd                  gdd g!      d7                      ZHd8 ZIee ed9 eJ ej                  eejd                  g eLd:                   ej                  dd g eMeN eLd:                  D  cg c]  \  } }d;j                  | |       c}} !      d<                      ZPd= ZQd> ZRe eej                  dk(  d?       ed@dAdBg      dC                      ZSedD        ZTdE ZUeedF               ZVyc c}} w )G    N)sleep)
with_numpynp)with_multiprocessing)with_dev_shm)raisesparametrizeskipif)make_memmap)Paralleldelayed)MemmappingPool)_TestingMemmappingExecutorhas_shareable_memory)ArrayMemmapForwardReducer)_strided_from_memmap)_get_temp_dir)_WeakArrayKeyMap_get_backing_memmapc                  2    t        j                  dd       y )Ni,  T)timeoutexit)faulthandlerdump_traceback_later     V/var/www/html/bid-api/venv/lib/python3.12/site-packages/joblib/test/test_memmapping.pysetup_moduler    !   s    %%c=r   c                  ,    t        j                          y N)r   cancel_dump_traceback_laterr   r   r   teardown_moduler$   %   s    ,,.r   c                      t        |       J | S r"   r   )arrays    r   check_memmap_and_send_backr'   )   s    u%111Lr   c                 V    | \  }}}t         j                  j                  ||   |       y)zDummy helper function to be executed in subprocesses

    Check that the provided array has the expected values in the provided
    range.

    Nr   testingassert_array_equalargsdatapositionexpecteds       r   check_arrayr1   .   s(      $D(HJJ!!$x.(;r   c                     | \  }}}||   |k(  sJ ||xx   dz  cc<   t         j                  j                  ||   d|z         y)zDummy helper function to be executed in subprocesses


    Check that the input array has the right values in the provided range
    and perform an inplace modification to double the values in the range by
    two.

       Nr)   r,   s       r   inplace_doubler4   9   sL      $D(H>X%%%NaNJJ!!$x.!h,?r   c                    t         j                  j                  }| j                  d      j                  }t        j
                  |t         j                  dd      }dt        j                  |j                  d   |j                        z  |dd |j                          t        j
                  |t         j                  d	d
dd      }t        j                  d      j                  |j                        |dd |ddddddf   }t        j                  |      }|j                  }t        d| j                  dd      fd} ||      }	t        |	      sJ t!        |	t         j
                        sJ  ||	|        ||      }
t        |
      sJ  ||
|        ||      }t!        |t         j
                        rJ t        |      sJ  |||        ||      }t!        |t         j
                        rJ t        |      sJ  |||       |dz  }t        |      rJ  ||      }t        |      rJ t!        |t         j
                        rJ  |||dz         t        j                  |      }t        |      rJ  ||      }t!        |t         j"                        sJ t        |      rJ  |||       y)z9Check that it is possible to reduce a memmap backed array	test.mmapi  w+dtypeshapemodeg      r   r9   N)         r+Fr?   )r9   r:   r;   orderoffset<      r3   cTc                 $     |       \  }} || S r"   r   xconsr-   reducers      r   reconstruct_array_or_memmapzEtest_memmap_based_array_reducing.<locals>.reconstruct_array_or_memmapi       QZ
dT{r   r=   )r   r*   r+   joinstrpathmemmapfloat64aranger:   r9   flushreshapeasarrayTr   r   
isinstancendarray)tmpdirr+   filenamebufferabrG   drM   a_reconstructedb_reconstructedc_reconstructedd_reconstructeda3a3_reconstructedb3b3_reconstructedrL   s                    @r    test_memmap_based_array_reducingrh   H   s}    66{{;'//H YYxrzz4HF 		&,,q/FFF1I
LLN 			("**I3q	2A99R=  )AaD
 	
!B$"ac/A 	

1A	A (fnnc4HG
 2!4O000oryy111* 2!4O000* 2!4O/299555000*1!4O/299555000* 
QB#B'''226#$4555*BII666'Q/ 
BB#B'''226&

333#$4555',r   win32r   z2PermissionError only easily triggerable on Windows)reasonc                    | j                  d      j                  }dj                  |      }t        j                  t
        j                  d|gt        j                  t        j                        }|j                          |j                         \  }}|j                  dk(  sJ |dk(  sJ dj                  |      }||j                         v sJ y )	Nr6   a  if 1:
    import os
    import numpy as np
    import time
    from joblib.externals.loky.backend import resource_tracker
    resource_tracker.VERBOSE = 1

    # Start the resource tracker
    resource_tracker.ensure_running()
    time.sleep(1)

    # Create a file containing numpy data
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    memmap[:] = np.arange(10).astype(np.int8).data
    memmap.flush()
    assert os.path.exists(r"{filename}")
    del memmap

    # Create a np.memmap backed by this file
    memmap = np.memmap(r"{filename}", dtype=np.float64, shape=10, mode='w+')
    resource_tracker.register(r"{filename}", "file")

    # Ask the resource_tracker to delete the file backing the np.memmap , this
    # should raise PermissionError that the resource_tracker will log.
    resource_tracker.maybe_unlink(r"{filename}", "file")

    # Wait for the resource_tracker to process the maybe_unlink before cleaning
    # up the memmap
    time.sleep(2)
    )r[   -cstderrstdoutr   r   z'tried to unlink {}, got PermissionError)rO   rP   format
subprocessPopensys
executablePIPEwaitcommunicate
returncodedecode)rZ   r[   cmdpouterrmsgs          r   2test_resource_tracker_retries_when_permissionerrorr      s     {{;'//H: 	!; < 	#..$4Z__ *	1AFFH}}HC<<1#::
3
:
:8
DC#**,r   c                    t         j                  j                  }| j                  d      j                  }t        j
                  |t         j                  dd      }t        j                  d      j                  |j                        |d d  |dd }|d d ddf   }|d d d d d d df   }|d	d
d   }t        d | j                  dd      fd} ||      }	t        |	      sJ t        |	t         j
                        sJ  ||	|        ||      }
t        |
      sJ  ||
|        ||      }t        |      sJ  |||        ||      }t        |      sJ  |||        ||      }t        |      sJ  |||       y )Nr6   )d      r   r=   r7   r8   i r   
   r>   rE   r=   r?   rG   Tc                 $     |       \  }} || S r"   r   rI   s      r   rM   zNtest_high_dimension_memmap_array_reducing.<locals>.reconstruct_array_or_memmap   rN   r   )r   r*   r+   rO   rP   rQ   rR   rS   rU   r:   r   r   rX   )rZ   r+   r[   r]   r^   rG   r_   erM   r`   ra   rb   rc   e_reconstructedrL   s                 @r   )test_high_dimension_memmap_array_reducingr      s    66{{;'//H 			("**4D	A99&'//8AaD 	
!BA	!QrT'
A	!Q1*A	!Aa%A (fnnc4HG 2!4O000oryy111*1!4O000*1!4O000*1!4O000*1!4O000*r   c                    | j                  d      j                  }dt        j                  z  }t        j                  dz   }t	        j
                  |d||z         }t        |dd|d|d d d	
	      }t        |t        j
                        sJ |j                  |k(  sJ t        |dd|d|dz  fd|d	
	      }t        |      j                  |k(  sJ y )Nr6   r>   rE   r7   )r;   r:   uint8rCF)r9   r;   rC   rB   r:   stridestotal_buffer_lenunlink_on_gc_collectr3   )r3   )
rO   rP   mmapALLOCATIONGRANULARITYr   rQ   r   rX   rC   r   )rZ   fnamesizerC   
memmap_objmemmap_backed_objs         r   test__strided_from_memmapr      s    KK$,,Et)))D''!+F5t4&=AJ%e7-33d.2T;@BJ j")),,,&&&,W3vSqylD4"
 0188FBBBr   factorymultiprocessingloky)idsc                 v   t         j                  j                  }|j                  d      j                  } | dd|      }	 |j                  d      j                  }t        j                  |t         j                  dd      }|j                  d	       |j                  t        t        |j                  d
         D cg c]&  }t        |j                  d         D ]	  }|||fd	f ( c}}        ||dt        j                  |j                        z         t        j                  |t         j                  dd      }	|j                  t        t        |	j                  d
         D cg c]&  }t        |	j                  d         D ]	  }|	||fdf ( c}}       t        j                  |      g k(  sJ  ||dt        j                  |j                        z          ||	dt        j                  |	j                        z         t        j                  |t         j                  ddd      }
t!        t"              5  |j                  t$        t        |
j                  d
         D cg c]  }|
|df	 c}       ddd       t!        t&        t(        f      5  |j                  t        t        |
j                  d
         D cg c]  }|
|df	 c}       ddd       |j+                          ~yc c}}w c c}}w c c}w # 1 sw Y   xY wc c}w # 1 sw Y   =xY w# |j+                          ~w xY w)z@Check that subprocess can access and update shared memory memmappoolr   r3   
max_nbytestemp_folderr6   r=   r>   r7   r8         ?r   rE   )r>   r=   rG   g       @r   r      )r9   r:   r;   rC   g      @N)r   r*   r+   mkdirrP   rO   rQ   float32fillmapr4   ranger:   onesoslistdirr   AssertionErrorr1   RuntimeError
ValueError	terminate)r   rZ   r+   pool_temp_folderr{   r[   r]   ijr^   rG   s              r   test_pool_with_memmapr   
  s    66 ||F+33q.>?A(;;{+33IIhbjjTJ	s	n(-aggaj(9;(91(-aggaj(91 !"Aq63/(9  0(9; 	< 	1a"''!''"223 IIhbjjSI	n(-aggaj(9;(91(-aggaj(91 !"Aq63/(9  0(9; 	< zz*+r111 	1a"''!''"2231a"''!''"223 IIhbjjC"$ N#EE+U1771:5FG5FAs5FGH $
 \:./EE.aggaj8I"J8I1Aq#;8I"JK 0 	
G;;"  H $# #K 0/ 	
s   A>L%  +K7+BL% .+K=B+L% +L/L;LL% +L
LLL% 7L% LLL% LL"L% %L8c                    t         j                  j                  }|j                  d      j                  } | dd|      }	 |j                  d      j                  }t        j                  |t         j                  dd      }|j                  d	       t        j                  |      }t        |t         j                        rJ t        |      sJ |j                  t        t        |j                  d
         D 	cg c]&  }t        |j                  d         D ]	  }	|||	fd	f ( c}	}        ||dt        j                   |j                        z          ||dt        j                   |j                        z         t#        j$                  |      g k(  sJ 	 |j'                          ~yc c}	}w # |j'                          ~w xY w)z?Check that subprocess can access and update shared memory arrayr   r   r3   r   r6   r   r7   r8   r   r   rE   N)r   r*   r+   r   rP   rO   rQ   r   r   rV   rX   r   r   r4   r   r:   r   r   r   r   )
r   rZ   r+   r   r{   r[   r]   a_viewr   r   s
             r    test_pool_with_memmap_array_viewr   @  sy    66 ||F+33q.>?A;;{+33IIhbjjTJ	s Afbii000#F+++	n(-aggaj(9;(91(-aggaj(91 !'A4(9  5(9; 	<
 	1a"''!''"22361rwwqww'7#78 zz*+r111 	
; 	
s    B<F6 >+F0)A4F6 0F6 6G	backendc                 h   dj                  |       }t        j                  t        j                  d|gt        j
                  t        j
                        }|j                          |j                         \  }}|j                  dk(  s)J |j                         dz   |j                         z          y )N  if 1:
        import numpy as np
        from joblib import Parallel, delayed


        data = np.random.rand(int(2e6)).reshape((int(1e6), 2))

        # Build a complex cyclic reference that is likely to delay garbage
        # collection of the memmapped array in the worker processes.
        first_list = current_list = [data]
        for i in range(10):
            current_list = [current_list]
        first_list.append(current_list)

        if __name__ == "__main__":
            results = Parallel(n_jobs=2, backend="{b}")(
                delayed(len)(current_list) for i in range(10))
            assert results == [1] * 10
    r^   rl   rm   r   

)
rp   rq   rr   rs   rt   ru   rv   rw   rx   ry   r   rz   r{   r|   r}   s        r   -test_permission_error_windows_reference_cycler   g  s    $ 	% & 	#..$4Z__ *	1AFFH}}HC<<1BcjjlV3cjjlBBr   c                    dj                  |       }t        d      D ]  }t        j                  j	                         }t        j
                  j                  t              |d<   t        j                  t        j                  d|gt        j                  t        j                  |      }|j                          |j                         \  }}|j                  dk(  sJ |       |dk(  sJ t        j                   d d d	vsd
|vrJ  y )Nak  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(int(2e6))

        if __name__ == '__main__':
            # warm-up call to launch the workers and start the resource_tracker
            _ = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(id)(i) for i in range(20))

            time.sleep(0.5)

            slice_of_data = Parallel(n_jobs=2, verbose=5, backend='{b}')(
                delayed(return_slice_of_data)(data, 0, 20) for _ in range(10))
    r   r=   
PYTHONPATHrl   rn   ro   envr   r   )r=      r   )r=   r   rE      resource_tracker)rp   r   r   environcopypathdirname__file__rq   rr   rs   rt   ru   rv   rw   rx   version_info)r   rz   _r   r{   r|   r}   s          r   3test_permission_error_windows_memmap_sent_to_parentr     s    ( 	) , 1XjjooGGOOH5LcnndC8$.OO$.OO> 	
==?S||q %#% czzBQ'==
 'c111 r   c                 d   t        j                  t        d             t        d| d      fdt	        d      D              \  } t        d| d      fdt	        d      D              \  }t
        j                  j                  |      t
        j                  j                  |      k7  sJ y )N      Y@r3   r   n_jobsr   r   c              3   J   K   | ]  } t        t              d         ywr[   Nr   getattr.0r   r&   s     r   	<genexpr>z6test_parallel_isolated_temp_folders.<locals>.<genexpr>  %      F5=
+X    #rE   c              3   J   K   | ]  } t        t              d         ywr   r   r   s     r   r   z6test_parallel_isolated_temp_folders.<locals>.<genexpr>  r   r   r   rS   intr   r   r   r   r   )r   
filename_1
filename_2r&   s      @r   #test_parallel_isolated_temp_foldersr     s     IIc#hEE81g"E F5:1XF LZ F81g"E F5:1XF LZ 77??:&"''//**EEEEr   c                 z   t        j                  t        d            t        d| d      5 } |fdt	        d      D              \  } |fdt	        d      D              \  }d d d        t
        j                  j                        t
        j                  j                        k(  sJ y # 1 sw Y   KxY w)Nr   r3   r   r   c              3   J   K   | ]  } t        t              d         ywr   r   r   s     r   r   z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>  $      
9AAGGUJ/r   rE   c              3   J   K   | ]  } t        t              d         ywr   r   r   s     r   r   z9test_managed_backend_reuse_temp_folder.<locals>.<genexpr>  r   r   r   )r   r{   r   r   r&   s       @r   &test_managed_backend_reuse_temp_folderr     s    
 IIc#hE	G	;q 
9>q
 
  
9>q
 
	 
< 77??:&"''//**EEEE 
<	;s   9B11B:c                     t        j                  t        d            } t               }t               }d }t	        j
                  || |f      }t	        j
                  || |f      }|j                          |j                          |j                          |j                          t        |      dk(  sJ t        |      dk(  sJ ||k7  sJ y )Nr   c                     t        ddd      5 }t        d      D ]L  } | fdt        d      D              \  }|j                  t        j                  j                  |             N 	 d d d        y # 1 sw Y   y xY w)Nr   r3   r   )r   r   r   c              3   J   K   | ]  } t        t              d         ywr   r   r   s     r   r   z]test_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename.<locals>.<genexpr>  s$      AIA$GG$UJ7r   rE   )r   r   addr   r   r   )r&   	temp_dirsr{   r   r[   s   `    r   concurrent_get_filenamezJtest_memmapping_temp_folder_thread_safety.<locals>.concurrent_get_filename  sc    fQ2>!2Y AFq 
 bggooh78	  ?>>s   AA55A>)targetr-   rE   )	r   rS   r   set	threadingThreadstartrO   len)r&   temp_dirs_thread_1temp_dirs_thread_2r   t1t2s         r   )test_memmapping_temp_folder_thread_safetyr     s     IIc#hE9 
		&e5G-H
B 
		&e5G-H
B HHJHHJGGIGGI!"a'''!"a'''!3333r   c            	         d} g d}|D ]  \  }}}t        j                  t        j                  d| j	                  ||      gt         j
                  t         j
                        }|j                          |j                         \  }}|j                  |k(  sJ |j                                d|vrJ |j                                 y )Na>  if 1:
        import os
        import numpy as np
        from joblib import Parallel, delayed
        from joblib.externals.loky.backend import resource_tracker
        from concurrent.futures import ThreadPoolExecutor, wait

        resource_tracker.VERBOSE = 0

        array = np.arange(int(1e2))

        temp_dirs_thread_1 = set()
        temp_dirs_thread_2 = set()


        def raise_error(array):
            raise ValueError


        def parallel_get_filename(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(getattr)(array, "filename") for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        def parallel_raise(array, temp_dirs):
            with Parallel(backend="loky", n_jobs=2, max_nbytes=10) as p:
                for i in range(10):
                    [filename] = p(
                        delayed(raise_error)(array) for _ in range(1)
                    )
                    temp_dirs.add(os.path.dirname(filename))


        executor = ThreadPoolExecutor(max_workers=2)

        # both function calls will use the same loky executor, but with a
        # different Parallel object.
        future_1 = executor.submit({f1}, array, temp_dirs_thread_1)
        future_2 = executor.submit({f2}, array, temp_dirs_thread_2)

        # Wait for both threads to terminate their backend
        wait([future_1, future_2])

        future_1.result()
        future_2.result()
    ))parallel_get_filenamer   r   )r   parallel_raiserE   )r   r   rE   rl   )f1f2rm   r   )
rq   rr   rs   rt   rp   ru   rv   rw   rx   ry   )rz   functions_and_returncodesr   r   rx   r{   r|   r}   s           r   ?test_multithreaded_parallel_termination_resource_tracker_silentr     s    1Cd! 8B
cnndCJJ"J4LM$.OOJOOM	==?S||z)73::<7)"#-;szz|;- 8r   c                    dj                  |       }t        j                  j                         }t        j                  j                  t              |d<   t        j                  t        j                  d|gt        j                  t        j                  |      }|j                          |j                         \  }}|j                  dk(  sJ |       |dk(  sJ t        j                  d d d	vrd
|vsJ y y )Na  if 1:
        import os
        import time

        import numpy as np

        from joblib import Parallel, delayed
        from testutils import return_slice_of_data

        data = np.ones(100)

        if __name__ == '__main__':
            for i in range(5):
                slice_of_data = Parallel(
                    n_jobs=2, max_nbytes=1, backend='{b}')(
                        delayed(return_slice_of_data)(data, 0, 20)
                        for _ in range(10)
                    )
    r   r   rl   r   r   r   r=   r   r   )rp   r   r   r   r   r   r   rq   rr   rs   rt   ru   rv   rw   rx   r   )r   rz   r   r{   r|   r}   s         r   'test_many_parallel_calls_on_same_objectr   L  s    $ 	% & **//
C1C	s#		A FFH}}HC<<1!c!#::
#99
 ##--- :r   c                     t        j                  t        d             t        d| d      fdt	        d      D              \  }t        |      J y )Ng     @@r3   r   r   c              3   H   K   | ]  } t        t                      y wr"   )r   r'   )r   r   r.   s     r   r   z8test_memmap_returned_as_regular_array.<locals>.<genexpr>  s%      CE;Ca+*+D18   "rE   )r   r   r   r   r   r   )r   resultr.   s     @r   %test_memmap_returned_as_regular_arrayr  |  sU     773s8DBxq'cB CE;@8CE EHVv&...r   c                    | dk(  r4t         j                  j                  d      rt        j                  d       dj                  |       }t        j                  t         j                  d|gt        j                  t        j                        }|j                          |j                         \  }}|j                         }|j                         }|j                  dk(  sJ |d	z   |z          d
|vsJ |       y )Nr   winzVThe temporary folder cannot be deleted on Windows in the presence of a reference cycler   r   rl   rm   r   r   resource_tracker)rs   platform
startswithpytestxfailrp   rq   rr   rt   ru   rv   rw   ry   rx   r   s        r   2test_resource_tracker_silent_when_reference_cyclesr
    s    " &S\\44U; 	,	

$ 	% & 	#..$4Z__ *	1AFFH}}HC
**,C
**,C<<10cFlS00S(-#-(r   c           	      Z   t        j                  |j                        g k(  sJ  | dd|j                  d      }	 t        j                  |j                        g k(  sJ t         j                  j	                  |j
                        rJ t        j                  dt        j                        }|j                  dk(  sJ |j                  t        t        |j                  d         D cg c]  }||d	f	 c}       t        j                  |j                        g k(  sJ t        j                  d
t        j                        }|j                  dk(  sJ |j                  t        t        |j                  d         D cg c]  }||d	f	 c}       t         j                  j                  |j
                        sJ t        j                  |j
                        }t!        |      dk(  sJ t        j"                  dgd
z  d      }|j                  t$        |g      }|d   rJ 	 |j'                          t        d      D ]9  }t)        d       t         j                  j	                  |j
                        r8 ~y t+        dj-                  |j
                              c c}w c c}w # |j'                          t        d      D ]9  }t)        d       t         j                  j	                  |j
                        r8 ~w  t+        dj-                  |j
                              xY w)z0Check that large arrays are not copied in memoryr=   (   r3   )r   r   verboser>   r<   r   r   r   r      rE   abcobjectr   皙?z#temporary folder {} was not deletedN)r   r   rP   r   exists_temp_folderr   r   r   nbytesr   r1   r   r:   rR   isdirr   r&   r   r   r   r   rp   )	r   rZ   r{   smallr   largedumped_filenamesobjectsresultss	            r   %test_memmapping_pool_for_large_arraysr    s    ::fnn%+++ 	bfnnaHA(zz&..)R///77>>!..111,||r!!!	kU5;;q>5JK5JUAsO5JKL zz&..)R/// 2::.||s"""	kU5;;q>5JK5JUAsO5JKL ww}}Q^^,,,::ann5#$))) ((E7S=9%%,wi81:~: 	
rA"I77>>!..1
  
 !5<<Q^^L = L L  	
rA"I77>>!..1
  
 !5<<Q^^L s3   B/J. +J$
7BJ. J)
BJ. $
J. .AL*(L*z,https://github.com/joblib/joblib/issues/1086)marksc                 n   dj                  |       }t        j                  j                         }t        j                  j                  t              |d<   t        j                  t        j                  d|gt        j                  t        j                  |      }|j                          |j                         \  }}|j                         |j                         }}|j                  d      d   }|j                   dk(  sJ |xs |       |dk(  sJ t        j                  j#                  |      rJ y )	Na  if 1:
        import os
        from pathlib import Path
        from time import sleep

        import numpy as np
        from joblib import Parallel, delayed
        from testutils import print_filename_and_raise

        data = np.random.rand(1000)

        def get_temp_folder(parallel_obj, backend):
            if "{b}" == "loky":
                return Path(parallel_obj._backend._workers._temp_folder)
            else:
                return Path(parallel_obj._backend._pool._temp_folder)


        if __name__ == "__main__":
            try:
                with Parallel(n_jobs=2, backend="{b}", max_nbytes=100) as p:
                    temp_folder = get_temp_folder(p, "{b}")
                    p(delayed(print_filename_and_raise)(data)
                              for i in range(1))
            except ValueError as e:
                # the temporary folder should be deleted by the end of this
                # call but apparently on some file systems, this takes
                # some time to be visible.
                #
                # We attempt to write into the temporary folder to test for
                # its existence and we wait for a maximum of 10 seconds.
                for i in range(100):
                    try:
                        with open(temp_folder / "some_file.txt", "w") as f:
                            f.write("some content")
                    except FileNotFoundError:
                        # temp_folder has been deleted, all is fine
                        break

                    # ... else, wait a bit and try again
                    sleep(.1)
                else:
                    raise AssertionError(
                        str(temp_folder) + " was not deleted"
                    ) from e
    r   r   rl   r   
r    )rp   r   r   r   r   r   r   rq   rr   rs   rt   ru   rv   rw   ry   splitrx   r  )r   rz   r   r{   r|   r}   r[   s          r   &test_child_raises_parent_exits_cleanlyr!    s    :-Z 	[ \ **//
C1C#..$4Z__ *S	:AFFH}}HCzz|SZZ\Cyyq!H<<1(cjS("99ww~~h''''r   c           	           | dd|j                         }	 t        j                  |j                         g k(  sJ t        j                  dt        j
                        }|j                  dk(  sJ |j                  t        t        |j                  d         D cg c]  }||df	 c}       t        j                  |j                         g k(  sJ 	 |j                          ~yc c}w # |j                          ~w xY w)	z2Check that large arrays memmapping can be disabledr=   Nr   r   r<   r  r   r   )rP   r   r   r   r   rR   r  r   r1   r   r:   r   )r   rZ   r{   r  r   s        r   .test_memmapping_pool_for_large_arrays_disabledr#  N  s     	d?A zz&..)R/// 2::.||s"""	kU5;;q>5JK5JUAsO5JKL zz&..)R/// 	
 L 	
s   BC* C%
'+C* %C* *C=c                    t         j                  }	 t        d      t         _         | dd      }	 |j                  }d}|j	                  |      sJ t
        j                  j                  |      sJ t        j                  dt        j                        }|j                  dk(  sJ |j                  t        |gdz         t        t        j                  |            d	k(  sJ t        j                  dt        j                        d
z  }|j                  dk(  sJ |j                  t        |gdz         t        t        j                  |            d
k(  sJ 	 |j!                          ~t#        d      D ].  }t
        j                  j                  |      s nt%        d       0 t'        d      	 |t         _        y# |j!                          ~w xY w# |t         _        w xY w)z1Check that memmapping uses /dev/shm when possibleg    ~Ar=   r   r   z"/dev/shm/joblib_memmapping_folder_r   r<   r  rE   r3   r  z(temporary folder of pool was not deletedN)jmrSYSTEM_SHARED_MEM_FS_MIN_SIZEr   r  r  r   r   r  r   r   rR   r  r   idr   r   r   r   r   r   )r   	orig_sizer{   r   folder_prefixr]   r^   r   s           r   'test_memmapping_on_large_enough_dev_shmr+  i  s    11I+6 -0I)A"%	  !~~@M#..}===77>>"2333 2::.A88s?"?EE"qcBh rzz"234999
 2::.2A88s?"?EE"qcBhrzz"234999 KKMsA77>>"23"I	  !!KLL 
 -6) KKM -6)s$   G D+G AG GG G%c                 l   t         j                  }	 t        d      t         _         | dd      }	 |j                  }|j	                  d      rJ 	 |j                          ~t        j                  j                  |      rJ 	 |t         _        y # |j                          ~w xY w# |t         _        w xY w)Ng@6Dr=   r   r%  z/dev/shm)	r&  r'  r   r  r  r   r   r   r  )r   r)  r{   r   s       r   $test_memmapping_on_too_small_dev_shmr-    s     11I6 -0J)A"%	  !~~'22:>>>> KKM 77>>"23333,5) KKM
 -6)s"   B& B 2B& B##B& &B3c                 f   t         j                  j                  } | dd|j                        }	 |j	                  t         j
                  d      }|j                         }t        |      rJ  ||t        j
                  d             |j                          ~y# |j                          ~w xY w)z:Check that large arrays are not copied in memory in returnr=   r   r   )  r-   r/  N)	r   r*   r+   rP   apply_asyncr   getr   r   )r   rZ   r+   r{   resr  s         r   /test_memmapping_pool_for_large_arrays_in_returnr4    s     66 	bfnn=AmmBGG'm2	'...5"''$-0	 	
s   AB B0c                 &    t        |       sJ | |z  S )z4Multiplication function to be executed by subprocessr   )r]   n_timess     r   _worker_multiplyr7    s    """w;r   c                    t         j                  j                  } | dd|j                        }	 t        j                  t        j
                  d      j                  d      d      dddd	ddf   }|j                  t        |df
      j                         }t        |      rJ  ||d|z         |j                          ~y# |j                          ~w xY w)zCheck that memmaps with a bad buffer are returned as regular arrays

    Unary operations and ufuncs on memmap instances return a new memmap
    instance with an in-memory buffer (probably a numpy bug).
    r=   r   r   ip  )r/  r3   r=   rA   )rB   NrE   r0  )r   r*   r+   rP   rV   rS   rU   r1  r7  r2  r   r   )r   rZ   r+   r{   r]   r^   s         r   6test_workaround_against_bad_memmap_with_copied_buffersr9    s     66bfnn=A JJryy..|< ""#RaR(,
 MM*!QM8<<>'***1a!e$	 	
s   BC Cc                     | S r"   r   )args    r   identityr<    s    Jr   zfactory,retry_nor=   z{}, {}c                    |j                  d      j                  }dt        j                  z  }t        j                  dz   }t	        |d|d|      } | d|j                        }|j                  t        |f	      j                         }t        |t        j                        sJ |j                  |k(  sJ t        j                  j                  ||       |j                          y )
Nr6   r>   rE   r7   r   )r;   r:   r9   rC   r3   r   r0  )rO   rP   r   r   r   r1  r<  r2  rX   r   rQ   rC   r*   r+   r   )	r   retry_norZ   r   r   rC   objr{   r  s	            r    test_pool_memmap_with_big_offsetrA    s     KK$,,Et)))D''!+F
e$d'#%C 	v~~.A]]83&]1557Ffbii(((==F"""JJ!!#v.KKMr   c                    d}t        || j                        \  }}|du sJ || j                  d      j                  k(  sJ t        |d       \  }}t        j                  j                  d      r|du sJ |j                  |      sJ y )Ntest.tmpdirFr>  r  )r   rP   rO   rs   r  r  endswith)rZ   pool_folder_namepool_folder
shared_mems       r   test_pool_get_temp_dirrH    s    $+,<fnnMK&++m4<<<<<+,<$OK
||u%U""" 0111r   c                 ,   d}ddl }t        |j                  j                  d      r&|j	                  |j                  j                  d       t        |d      \  }}t        j                  j                  d      r|du sJ |j                  |      sJ y)z`Check that _get_temp_dir works when os.statvfs is not defined

    Regression test for #902
    rC  r   Nstatvfsr>  r  F)
joblib._memmapping_reducerhasattr_memmapping_reducerr   delattrr   rs   r  r  rD  )rZ   monkeypatchrE  joblibrF  rG  s         r   !test_pool_get_temp_dir_no_statvfsrQ  $  s    
 %%v)),,i8F66999E+,<$OK
||u%U""" 0111r   z1This test fails with a PermissionError on Windows	mmap_moder@   r7   c                 "   d t        d      D cg c]  }t        j                  dd       }} t        | dd      fd	t	        |      D              }t	        |      D ]%  \  }}t        j
                  j                  ||       ' y c c}w )
Nc                     || d d  | S r"   r   )arrvalues     r   funcz4test_numpy_arrays_use_different_memory.<locals>.func:  s    A
r   r   )r   r   rR   r<   r   r3   )rR  r   r   c              3   H   K   | ]  \  }} t              ||        y wr"   r   )r   r   rU  rW  s      r   r   z9test_numpy_arrays_use_different_memory.<locals>.<genexpr>@  s(      D?,=&!Sc1,=r   )r   r   zerosr   	enumerater*   r+   )rR  r   arraysr  rU  rW  s        @r   &test_numpy_arrays_use_different_memoryr]  5  s    
 <A9E9abhhxy19FEChqC D?,5f,=D? ?G G$3


%%c1- % Fs   Bc            	         dd} t        j                  d      }t               }|j                  |d       |j	                  |      dk(  sJ |}|j	                  |      dk(  sJ |j                  |d       |j	                  |      dk(  sJ ~t        j                          t        |j                        dk(  sJ |j	                  |      dk(  sJ ~ | |j                         t        j                  d      }|j                  |d       t        |j                        dk(  sJ |j	                  |      dk(  sJ t        t              5  |j	                  t        j                  d             d d d        ~ | |j                         d }t        t        d	      D cg c]  } |||       c}      }t        j                         d
k(  r0t        t        j                   dd      rdnd}t        |      |k  sJ y y # 1 sw Y   xY wc c}w )Nr   c                     t        |      D ]1  }t        |       dk(  r y t        j                          t	        d       3 t        |       dk(  sJ y )Nr   r  )r   r   gccollectr   )	containerretriesr   s      r   assert_empty_after_gc_collectz>test_weak_array_key_map.<locals>.assert_empty_after_gc_collectJ  sC    wA9~"JJL"I	  
 9~"""r   *   r]   r^   rE   rG   c                     t        j                  d      }t        t              5  | j	                  |       d d d        | j                  ||       | j	                  |      |k(  sJ t        |      S # 1 sw Y   <xY w)Nre  )r   r   r   KeyErrorr2  r   r(  )mr   r]   s      r   get_set_get_collectz4test_weak_array_key_map.<locals>.get_set_get_collectq  sZ    GGBKHEE!H 	auuQx1}}!u	 s   A22A;r/  CPythonnogilFi  )r   )r   r   r   r   r2  r`  ra  r   _datar   rg  r   r  python_implementationr   rs   flags)	rd  r]   rh  r^   rG   ri  r   
unique_idsmax_len_unique_idss	            r   test_weak_array_key_maprq  G  s   # 	AAEE!SM558s??	A558s??EE!SM558s??	JJLqww<1558s??	!!''*
AEE!SMqww<1558s??			bggbk 
 	
!!''* tEA)!Q/EFJ%%'94 %,CIIw$FSC:!3333 5# 
	  Fs   
%G>H
>Hc                      t               } t        t        j                        5  t        j                  |        d d d        y # 1 sw Y   y xY wr"   )r   r   picklePicklingErrordumps)rh  s    r   #test_weak_array_key_map_no_picklingrv    s/    A	$$	%Q 
&	%	%s   AAc                    t        | j                  d            t        j                  dd      }|j	                         fdd         } t        d      fd	|fD              } t        d      fd
|fD              }t        j                  j                  ||       fd t        d      fdt        d      D              }t        j                  j                  |d   |       y )Nzarr.datr   r   r<   c                      t              5 } t        j                  | j                         dt        j                  d      }d d d        t	        j
                  dt        j                  d      S # 1 sw Y   0xY w)Nr   )accessrC   r   )r9   r\   rC   )openr   filenoACCESS_READr   rY   r   )fdmmtestfiles     r   _read_arrayz%test_direct_mmap.<locals>._read_array  sR    (^r299;$2B2B1MB zz%rxx1EE ^s   6A22A;c                     | dz  S )Nr3   r   )rJ   s    r   rW  ztest_direct_mmap.<locals>.func  s    !tr   r3   )r   c              3   @   K   | ]  } t              |        y wr"   rY  r   rJ   rW  s     r   r   z#test_direct_mmap.<locals>.<genexpr>  s     ;s!]WT]1-s   c              3   @   K   | ]  } t              |        y wr"   rY  r  s     r   r   z#test_direct_mmap.<locals>.<genexpr>  s      A5aq!15r  c                               S r"   r   )r  s   r   workerz test_direct_mmap.<locals>.worker  s    }r   c              3   >   K   | ]  } t                       y wr"   rY  )r   r   r  s     r   r   z#test_direct_mmap.<locals>.<genexpr>  s      EHq!2Hs   rE   r   )	strrO   r   rS   tofiler   r*   r+   r   )	rZ   r]   rU  refr  r  rW  r  r  s	        @@@@r   test_direct_mmapr    s     6;;y)*H
		"G$AHHXF
 -C (!
;s;
;C !ha  AC5 AAGJJ!!'3/ !ha  EE!H EEGJJ!!'!*c2r   )Wr   r   rs   r  r`  rs  	itertoolstimer   rq   r   r   r  joblib.test.commonr   r   r   r   joblib.testingr   r	   r
   joblib.backportsr   joblib.parallelr   r   joblib.poolr   joblib.executorr   TestExecutorrK  r   r   r   r   r   r   rM  r&  r    r$   r'   r1   r4   rh   r   r   r   get_memmapping_executorr   r   r   r   r   r   r   r   r   r  r
  r  parammarkr	  r!  r#  r+  r-  r4  r7  r9  r<  listproductr   r   r  rp   rA  rH  rQ  r]  rq  rv  r  )rJ   ys   00r   <module>r     sL   	  
  	        - 3 + 6 6 ( - & F ; @ ; 4 7 : (>/
<@ I-  I-X 	 'RCE)E )X *+  *+Z C C* Y)M)MN#V,./.  /d Y)M)MN#V,. .   F Y*F34C 5  C@ Y*F34,2 5  ,2^ Y*F34
F 5  
F Y*F34
F 5  
F #4  #4L C<  C<L Y*F34*. 5  *.Z Y*F34/ 5  / Y*F341. 5  1.h Y)M)MN#V,.1.  1h ++##E $ 	
 	G(  G(T Y)M)MN#V,..  . Y)M)MN#V,..6.   
.6b Y)M)MN#V,.6.   
60 Y)M)MN#V,..  , Y)M)MN#V,..  2 				==>a
J K+<9+<+<	F#SeAh%7,9 	: ,941aA	 ,9 	:	;;  &	22"  )% &[4,'. (& . 94 94x 3  3S	:s   6P8