
    =[g                     J   d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	m
Z
mZmZmZmZmZ ddlZddlmZmZmZmZmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ d
ede fdZ!de
dee    dee e	f   fdZ" G d de      Zdede#dee e	f   fdZ$dede#dee e	f   fdZ%dede fdZ&defdZ'd,de deee e#f      dejP                  fdZ)de dee#   fdZ*dede#fdZ+dede,fdZ-dede,fd Z.d!ede#fd"Z/de fd#Z0d$e d%e
g ef   defd&Z1d'ede fd(Z2d$e defd)Z3d*ee    de,fd+Z4y)-z;Xgboost pyspark integration submodule for helper functions.    N)Thread)AnyCallableDictOptionalSetTypeUnion)BarrierTaskContext	SparkConfSparkContext
SparkFilesTaskContext)SparkSession)BoosterXGBModel)CommunicatorContext)RabitTrackerclsreturnc                 8    | j                    d| j                   S )zReturn the class name..)
__module____name__)r   s    N/var/www/html/bid-api/venv/lib/python3.12/site-packages/xgboost/spark/utils.pyget_class_namer      s    nnQs||n--    funcunsupported_setc                     t        j                  |       }i }|j                  j                         D ]C  }|j                  |j
                  us|j                  |vs+|j                  ||j                  <   E |S )zReturns a dictionary of parameters and their default value of function fn.  Only
    the parameters with a default value will be included.

    )inspect	signature
parametersvaluesdefaultemptyname)r   r   sigfiltered_params_dict	parameters        r   _get_default_params_from_funcr+      sn     

D
!C^^**,	 Y__4o53<3D3D 0 -  r   c                   0     e Zd ZdZdededdf fdZ xZS )r   z&Context with PySpark specific task ID.contextargsr   Nc                 \    t        |j                               |d<   t        |   di | y )Ndmlc_task_id )strpartitionIdsuper__init__)selfr-   r.   	__class__s      r   r5   zCommunicatorContext.__init__1   s+    "7#6#6#89^ 4 r   )r   r   __qualname____doc__r   r   r5   __classcell__)r7   s   @r   r   r   .   s&    0! 2 !C !D ! !r   r   r-   	n_workersc                     d|i}t        |       }t        ||d      }|j                          t        |j                        }d|_        |j                          |j                  |j                                |S )z"Start Rabit tracker with n_workersr;   task)r;   host_ipsortby)targetT)_get_host_ipr   startr   wait_fordaemonupdateworker_args)r-   r;   r.   hosttrackerthreads         r   _start_trackerrJ   6   sg    '3D DYVLGMMO7++,FFM
LLNKK##%&Kr   c                     t        | |      }|S )z3Get rabit context arguments to send to each worker.)rJ   )r-   r;   envs      r   _get_rabit_argsrM   C   s    
)
,CJr   c                     | j                         D cg c]   }|j                  j                  d      d   " }}|d   S c c}w )zLGets the hostIP for Spark. This essentially gets the IP of the first worker.:r   )getTaskInfosaddresssplit)r-   infotask_ip_lists      r   rA   rA   I   sE    ;B;O;O;QR;Q4DLL&&s+A.;QLR? Ss   %?c                      t         j                  j                         t        d      t        j
                  j                         S )z`Get or create spark session. Note: This function can only be invoked from driver
    side.

    z<_get_spark_session should not be invoked from executor side.)pysparkr   getRuntimeErrorr   buildergetOrCreater1   r   r   _get_spark_sessionr[   O   s@    
  ,J
 	
 ++--r   r'   levelc                    t        j                  |       }||j                  |       n<|j                  t         j                  k(  r|j                  t         j
                         |j                  sxt        j                         j                  sZt        j                  t        j                        }t        j                  d      }|j                  |       |j                  |       |S )zGGets a logger by name, or creates and configures it for the first time.z<%(asctime)s %(levelname)s %(name)s: %(funcName)s %(message)s)logging	getLoggersetLevelr\   NOTSETINFOhandlersStreamHandlersysstderr	FormattersetFormatter
addHandler)r'   r\   loggerhandler	formatters        r   
get_loggerrm   \   s    t$F <<7>>)OOGLL)??7#4#4#6#?#?''

3%%J
	 	Y''"Mr   c                     t        j                  |       }|j                  t         j                  k(  rdS |j                  S )z+Get the logger level for the given log nameN)r^   r_   r\   ra   )r'   rj   s     r   get_logger_levelro   p   s0    t$F<<7>>14Cv||Cr   spark_contextc                 d   | j                   j                         j                         dk\  r^| j                   j                         j                  | j                   j                         j	                         j                  d            S | j                   j                         j                         S )z0Gets the current max number of concurrent tasks.z3.1r   )_jscscversionmaxNumConcurrentTasksresourceProfileManagerresourceProfileFromIdrp   s    r   _get_max_num_concurrent_tasksry   v   s     &&(E1!!$$&<<!!#::<RRSTU
 	
   "88::r   c                 R    | j                   j                         j                         S )zWhether it is Spark local mode)rr   rs   isLocalrx   s    r   	_is_localr|      s"       "**,,r   confc                 x    | j                  d      }|d uxr$ |j                  d      xs |j                  d      S )Nzspark.masterzspark://zlocal-cluster)rW   
startswith)r}   masters     r   _is_standalone_or_localclusterr      s@    XXn%F *%K):):?)Kr   task_contextc                     | t        d      | j                         }d|vrt        d      t        |d   j                  d   j	                               S )z&Get the gpu id from the task resourcesz3_get_gpu_id should not be invoked from driver side.gpuzDCouldn't get the gpu id, Please check the GPU resource configurationr   )rX   	resourcesint	addressesstrip)r   r   s     r   _get_gpu_idr      s`    PQQ&&(IIR
 	
 y))!,22455r   c                      t        j                         } t        j                  j	                  | d      }t        j                  j                  |      st        j                  |       |S )Nzxgboost-tmp)r   getRootDirectoryospathjoinexistsmakedirs)root_dirxgb_tmp_dirs     r   _get_or_create_tmp_dirr      sF    **,H'',,x7K77>>+&
K r   modelxgb_model_creatorc                 f     |       }|j                  t        | j                  d                   |S )zH
    Deserialize an xgboost.XGBModel instance from the input model.
    utf-8)
load_model	bytearrayencode)r   r   	xgb_models      r   deserialize_xgb_modelr      s.     "#I5<<#89:r   boosterc                    t         j                  j                  t               t	        j
                          d      }| j                  |       t        |d      5 }|j                         }ddd       |S # 1 sw Y   S xY w)z
    Serialize the input booster to a string.

    Parameters
    ----------
    booster:
        an xgboost.core.Booster instance
    .jsonr   encodingN)	r   r   r   r   uuiduuid4
save_modelopenread)r   tmp_file_namefser_model_strings       r   serialize_boosterr      sf     GGLL!7!9djjl^5;QRM}%	mg	.!668 
/ 
/s   A77Bc                    t               }t        j                  j                  t	               t        j                          d      }t        |dd      5 }|j                  |        ddd       |j                  |       |S # 1 sw Y   xY w)zN
    Deserialize an xgboost.core.Booster from the input ser_model_string.
    r   wr   r   N)
r   r   r   r   r   r   r   r   writer   )r   r   r   r   s       r   deserialize_boosterr      sj     iGGGLL!7!9djjl^5;QRM	mS7	3q	 
4}%N 
4	3s   BBdevicec                 
    | dv S )z&Whether xgboost is using CUDA workers.)cudar   r1   )r   s    r   use_cudar      s    _$$r   )N)5r9   r!   r^   r   re   r   	threadingr   typingr   r   r   r   r   r	   r
   rV   r   r   r   r   r   pyspark.sql.sessionr   xgboostr   r   xgboost.collectiver   CCtxxgboost.trackerr   r2   r   r+   r   rJ   rM   rA   r[   Loggerrm   ro   ry   boolr|   r   r   r   r   r   r   r   r1   r   r   <module>r      s   A   	 
   B B B  X X , % : (. . .
 
 %(X 	#s(^ &!$ !
. 
3 
4S> 
/ C DcN ,  
.L 
.S %S/!: gnn (D3 D8C= D; ;# ;-\ -d - t 6k 6c 6 #+BL#9w 3 "
s 
w 
%Xc] %t %r   