bcg6ddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z m Z ejeZddZdZdZdZdadZd Zdd Zdd Zdd Zd ZdZdZdZdZdZ dZ!dZ"dZ#dZ$dZ%y)N) PATH_TYPESfspathc,tj}tj| tj|d5}|j dddtj|y#1swYxYw#tj|wxYw)Nzr:gz)osgetcwdchdirtarfileopen extractall)src_filedst_dircur_dirfs L/var/www/html/bid-api/venv/lib/python3.12/site-packages/catboost/datasets.py_extractrsbiikGHHW \\(F +q LLN, , + s#A<A0A<0A95A<<Bctj}t|d5} |jd}|sn|j |& ddd|j S#1swY|j SxYw)Nrbi)hashlibmd5r readupdate hexdigest)pathhasherrblocks r _calc_md5rsr [[]F dD QFF5ME MM%           s (A""A:c tj|y#t$r#tjj |sYywxYwN)rmakedirsOSErrorrisdirrs r_ensure_dir_existsr$&s: D ww}}T" #s)AAcFtjj|rt||k(ryd}t |t st |t r|n|f}|D]:} tjjjj|||n t#d|t|}||k7rt#dj%|||y#tjjjjtf$rtj!d|YwxYw)Nc0tjd|y)Nzdownloaded %s bytes)loggerdebug)blocknumbssizes r reporthookz$_cached_download..reporthook2s *D1)r,zfailed to download from %szBmd5 sum mismatch for url {url}; expected {expected}, but got {got})urlexpectedgot)rrisfiler isinstancelisttuplesixmovesurllibrequest urlretrieveerrorURLErrorIOErrorr'r( RuntimeErrorformat)r.rdstr,urlsudst_md5s r_cached_downloadrC.s ww~~cy~42S$':c5+A3wD  : II   $ $ 0 0CJ 0 O 7>>nG#~_ffCWg./ /    &&//9 : LL5q 9 :s6CA D D c~t2tjjtjdatS)Ncatboost_cached_datasets) _cache_pathrrjoinrr-r_get_cache_pathrIIs)ggll299;0JK r-cHt|tsJdt|ay)Nzexpected string or pathlib.Path)r2rrrFr#s rset_cache_pathrKPs! dJ 'J)JJ ',Kr-c|r(tjjt|nt j }tjj||}tjj||}tjj |rtjj |sft|t j\} } tj|  t||| t| |tj| |st j\} } t j\} }tj| tj| tj|| tj||tj|| |}}||fS#tj| wxYwr)rrrGrItempfilemkdtempexistsr$mkstempcloserCrremovereplaceshutilrmtree)r.r dataset_name train_file test_filecachedir_path train_path test_pathfile_descriptor file_path fd_new_trainnew_train_path fd_new_test new_test_paths r_download_datasetrcVsL AFrww||O-|<8K[K[K]Hh 3J Xy1I GGNN: &277>>)+D8$%-%5%5%7" ! ! S#y 1 Y ) IIi '/'7'7'9$ n%-%5%5%7" ]   :~. 9m, h . I y   IIi s 5GGct||||||\}} tj|||tj| ||} } |s*tj|tj| | | fS)N)headersep)rcpdread_csvrrR) r.rrVrWrXrfrerYr[r\traintests r_load_dataset_pdrktsg-c3 jR[]bcJ ++jSA2;;yagmpCq4E  * ) $;r-ctj||ftjd}t|d5}t |D]\\}}tj |tj|}|j |k(sJd|dz||j fz|||dd^ ddddz|k(sJd||dzfztj|S#1swY5xYw)NF)dtypeorderr)rnrfz=got too many columns at line %d (expected %d columns, got %d)rz.got too many lines (expected %d lines, got %d)) npzerosfloat32r enumerate fromstringr+rg DataFrame) r row_count column_countrfdatasetrline_idxlinerows r_load_numeric_only_datasetr|}shh <2"**CPG dD Q'lNHd--BJJC@C88|+ V-lpx{|p|KMPMUMUpV.V V+$'GH a +  a<9 $r&VZcempqeqYr&rr $ <<    s A+CCc4d}d}d\}}}t|||||S)NzLhttps://storage.mds.yandex.net/get-devtools-opensource/233854/titanic.tar.gz 9c8bc61d545c6af244a1d37494df3fc3)titanic train.csvtest.csvrkr.rrVrWrXs rrrs+ XC ,C*L'L*i ClJ JJr-c4d}d}d\}}}t|||||S)NzKhttps://storage.mds.yandex.net/get-devtools-opensource/250854/amazon.tar.gz 8fe3eec12bfd9c4c532b24a181d0aa2c)amazonrrrrs rrrs+ WC ,C*K'L*i ClJ JJr-c <d}d}d\}}}t|||||dddS)NzKhttps://storage.mds.yandex.net/get-devtools-opensource/233854/msrank.tar.gz 34fee225d02419adc106581f4eb36f2e)msrank train.tsvtest.tsv T)rerfrYrrs rrrs4 WC ,C*K'L*i ClJ RV\`hl mmr-c8d}d}d\}}}t|||||dS)NzOhttps://storage.mds.yandex.net/get-devtools-opensource/250854/msrank_10k.tar.gz 79c5b67397289c4c8b367c1f34629eae) msrank_10krr)rerrs rrrs. [C ,C*O'L*i ClJ RV WWr-c8d}d}d\}}}t|||||dS)a Contains information from kaggle [1], which is made available here under the Open Database License (ODbL) [2]. Download "rotten_tomatoes" [1] data set. Will return two pandas.DataFrame-s, first with train part (rotten_tomatoes.data) and second with test part (rotten_tomatoes.test) of the dataset. NOTE: This is a preprocessed version of the dataset. [1]: https://www.kaggle.com/rpnuser8182/rotten-tomatoes [2]: https://opendatacommons.org/licenses/odbl/1-0/index.html z@https://catboost-opensource.s3.yandex.net/rotten_tomatoes.tar.gz a07fed612805ac9e17ced0d82a96add4)rotten_tomatoes learn.tsvrrrfrrs rrrs0 MC ,C*T'L*i ClJ t TTr-c8d}d}d\}}}t|||||dS)Nz5https://catboost-opensource.s3.yandex.net/imdb.tar.gz 0fd62578d631ac3d71a71c3e6ced6f8b)imdbrrrrrrs rrrs- AC ,C*I'L*i ClJ t TTr-czd}d}d\}}}t|||||d\}}t|ddd t|d dd fS) a Download "epsilon" [1] data set. Will return two pandas.DataFrame-s, first with train part (epsilon_normalized) and second with test part (epsilon_normalized.t) of the dataset. Object class will be located in the first column of dataset. NOTE: This is a preprocessed version of the dataset. It was converted from libsvm format into tsv (CatBoost doesn't support libsvm format out of the box). [1]: https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary.html#epsilon )z.https://proxy.sandbox.yandex-team.ru/785711439zLhttps://storage.mds.yandex.net/get-devtools-opensource/250854/epsilon.tar.gz 5bbfac403ac673da7d7ee84bd532e973)epsilonrrTrYiirrircr|)r@rrVrWrXr[r\s rrrs` ZD -C*L'L*i-dCzS\dhiJ ":vtF"9fdE GGr-c :d}d}d\}}}t|||||ddS)a Dataset with monotonic constraints. Can be used for poisson regression. Has several numerical and several categorical features. The first column contains target values. Columns with names Cat* contain categorical features. Columns with names Num* contain numerical features. Dataset also contains several numerical features, for which monotonic constraints must hold. For features in columns named MonotonicNeg*, if feature value decreases, then prediction value must not decrease. Thus, if there are two samples x1, x2 with all features being equal except for a monotonic negative feature M, such that x1[M] > x2[M], then the following inequality must hold for predictions: f(x1) <= f(x2) zOhttps://storage.mds.yandex.net/get-devtools-opensource/479623/monotonic1.tar.gz 1b9d8e15bc3fd6f1498e652e7fc4f4ca) monotonic1rrrT)rfrYrrs rrrs3 \C ,C*O'L*i ClJ t[_ ``r-c8d}d}d\}}}t|||||dS)a Dataset with monotonic constraints. Can be used for regression. The first column contains target values. Other columns contain contain numerical features, for which monotonic constraints must hold. For features in columns named MonotonicNeg*, if feature value decreases, then prediction value must not decrease. Thus, if there are two samples x1, x2 with all features being equal except for a monotonic negative feature MNeg, such that x1[MNeg] > x2[MNeg], then the following inequality must hold for predictions: f(x1) <= f(x2) For features in columns named MonotonicPos*, if feature value decreases, then prediction value must not increase. Thus, if there are two samples x1, x2 with all features being equal except for a monotonic positive feature MPos, such that x1[MPos] > x2[MPos], then the following inequality must hold for predictions: f(x1) >= f(x2) zOhttps://storage.mds.yandex.net/get-devtools-opensource/250854/monotonic2.tar.gz ce559e212cb72c156269f6f9a641baca) monotonic2rrrrrrs rrrs0 \C ,C*O'L*i ClJ t TTr-cd}tttttttttttttttd}d}d}tj\}}t j |t |||d}d}tj\}} t j |t ||| tj||dddgd } t j|tj| |dddgd d d id } t j| | j|} | j|} | | fS)a  Download "Adult Data Set" [1] from UCI Machine Learning Repository. Will return two pandas.DataFrame-s, first with train part (adult.data) and second with test part (adult.test) of the dataset. [1]: https://archive.ics.uci.edu/ml/datasets/Adult )age workclassfnlwgt educationz education-numzmarital-status occupation relationshipracesexz capital-gainz capital-losszhours-per-weekznative-countryincome)z.https://proxy.sandbox.yandex-team.ru/779118052zJhttps://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.data 5d7c39d7b8804f071cdd1f2a7c460872)z.https://proxy.sandbox.yandex-team.ru/779120000zJhttps://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.test 35238206dfdf7f1fe215bbb874adecdcNz,\s*?python)namesrerf na_valuesenginerrc |ddS)NrH)xs rzadult..5sJKLOMOJPr-)rrerfrskiprows convertersr) floatobjectrMrPrrQrCrgrhrRastype) rrn train_urls train_md5fd_trainr[ test_urlstest_md5fd_testr\train_dftest_dfs radultr sA &E 6U&vu F  6EXJ3I#++-HjHHXZJ7XI2H!))+GYHHWY)4{{:U4WY\X]fnoHIIjkk)57WZV[fgu}@PuQZbcGIIiu%HnnU#G W r-czd}d}d\}}}t|||||d\}}t|ddd t|d dd fS) a Download "higgs" [1] data set. Will return two pandas.DataFrame-s, first with train part and second with test part of the dataset. Object class will be located in the first column of dataset. [1]: https://archive.ics.uci.edu/ml/datasets/HIGGS zJhttps://storage.mds.yandex.net/get-devtools-opensource/250854/higgs.tar.gz ad59ba8328a9afa3837d7bf1a0e10e7b)higgsrrTri7rri r)r.rrVrWrXr[r\s rrr@s^ WC ,C*J'L*i-c3 jR[cghJ ":xF"9fbdC EEr-).)F),inferF)r)&rloggingnumpyrprpandasrgr rMr5rTcorerr getLogger__name__r'rrr$rCrFrIrKrcrkr|rrrrrrrrrrrrHr-rrs  $   8 $/0  !<!&KKnXU(UG0a(U,4nEr-