a
    nÝEbÏ”  ã                   @   s  d Z ddlZddlmZ ddlZddlZddlmZmZ ddl	Z
ddlZddlmZ ddlmZmZmZ ddlm  mZ ddlZddlmZ ddlmZ ddlmZmZm Z m!Z!m"Z" zddl#Z#d	Z$W n e%yÜ   d
Z$Y n0 zTeƒ �4 edde&d� edde'd� ddl(Z(W d  ƒ n1 �s"0    Y  d	Z)W n e%�yJ   d
Z)Y n0 ej* d¡Z+ej,ej-dej*j.e) �pzedƒdkdd�d�ej-dej*j.e$ dd�d�gd�dd„ ƒZ/ej,dd„ ƒZ0ej,dd„ ƒZ1ej,dd „ ƒZ2ej,d!d"„ ƒZ3ej,d#d$„ ƒZ4ej,ej 5ej6j7¡ej 5ej6j8¡ej 5ej6j9¡ej :d%d&¡ej :d'd&¡ej :d(d&¡ej :d)d&¡gd�d*d+„ ƒZ;dJd-d.„Z<d/d0„ Z=d1d2„ Z>d3d4„ Z?d5d6„ Z@d7d8„ ZAd9d:„ ZBd;d<„ ZCd=d>„ ZDd?d@„ ZEG dAdB„ dBƒZFG dCdD„ dDeFƒZGej* dE¡G dFdG„ dGeFƒƒZHG dHdI„ dIeFƒZIdS )Kz test parquet compat é    N)ÚBytesIO)Úcatch_warningsÚfilterwarnings)Ú
get_option)Úpa_version_under2p0Úpa_version_under5p0Úpa_version_under6p0)ÚVersion)ÚFastParquetImplÚPyArrowImplÚ
get_engineÚread_parquetÚ
to_parquetTFÚignorez	`np.bool`)Úcategoryz.*Int64Index.*z4ignore:RangeIndex.* is deprecated:DeprecationWarningÚfastparquetúmode.data_managerÚarrayz4fastparquet is not installed or ArrayManager is used©Úreason)ZmarksÚpyarrowúpyarrow is not installed)Úparamsc                 C   s   | j S ©N©Úparam©Úrequest© r   ú\/home/ja/django-apps/lartica_env/lib/python3.9/site-packages/pandas/tests/io/test_parquet.pyÚengine@   s    r    c                   C   s   t st d¡ dS )Nr   r   )Ú_HAVE_PYARROWÚpytestÚskipr   r   r   r   ÚpaU   s    
r$   c                   C   s*   t st d¡ ntdƒdkr&t d¡ dS )Nzfastparquet is not installedr   r   z.ArrayManager is not supported with fastparquetr   )Ú_HAVE_FASTPARQUETr"   r#   r   r   r   r   r   Úfp\   s
    
r&   c                   C   s   t  g d¢ddœ¡S )N©é   é   é   Úfoo)ÚAÚB)ÚpdÚ	DataFramer   r   r   r   Ú	df_compate   s    r0   c               
   C   sB   t  tdƒttddƒƒtjdddd�g d¢t jd	d
d�dœ¡} | S )NÚabcr(   é   ç      @ç      @Úfloat64©Údtype©TFTÚ20130101r*   ©Úperiods)ÚaÚbÚdÚeÚf)r.   r/   ÚlistÚrangeÚnpÚarangeÚ
date_range)Údfr   r   r   Údf_cross_compatj   s    úÿrG   c                   C   sŠ   t  tdƒdtjdgg d¢g d¢g d¢ttddƒƒt d	d
¡ d¡tjdddd�dtjdgg d¢t jdd	d�t  	d¡t j
t  	d¡gdœ¡S )Nr1   r<   Úc©r<   NrH   )ó   fooó   bars   baz©r+   ÚbarÚbazr(   r2   r*   é   Úu1r3   r4   r5   r6   ç       @g      @r8   r9   r:   Z20130103)ÚstringZstring_with_nanZstring_with_noneÚbytesÚunicodeÚintZuintÚfloatZfloat_with_nanÚboolÚdatetimeZdatetime_with_nat)r.   r/   rA   rC   ÚnanrB   rD   ÚastyperE   Ú	TimestampZNaTr   r   r   r   Údf_full|   s$    

ýôÿr\   z2019-01-04T16:41:24+0200z%Y-%m-%dT%H:%M:%S%zz2019-01-04T16:41:24+0215z2019-01-04T16:41:24-0200z2019-01-04T16:41:24-0215c                 C   s   | j S r   r   r   r   r   r   Útimezone_aware_date_list”   s    r]   r)   c
                    s”   ˆp
ddi‰ˆpi ‰ˆdu r ˆ‰|r4|ˆd< |ˆd< ‡ ‡‡‡‡‡‡‡fdd„}
ˆdu rˆt  ¡ �‰|
|	ƒ W d  ƒ q�1 s|0    Y  n|
|	ƒ dS )a  Verify parquet serializer and deserializer produce the same results.

    Performs a pandas to disk and disk to pandas round trip,
    then compares the 2 resulting DataFrames to verify equality.

    Parameters
    ----------
    df: Dataframe
    engine: str, optional
        'pyarrow' or 'fastparquet'
    path: str, optional
    write_kwargs: dict of str:str, optional
    read_kwargs: dict of str:str, optional
    expected: DataFrame, optional
        Expected deserialization result, otherwise will be equal to `df`
    check_names: list of str, optional
        Closed set of column names to be compared
    check_like: bool, optional
        If True, ignore the order of index & columns.
    repeat: int, optional
        How many times to repeat the test
    ÚcompressionNr    c              	      sr   t | ƒD ]d}ˆjˆfi ˆ¤Ž tdd��  tˆfi ˆ¤Ž}W d   ƒ n1 sN0    Y  tjˆ|ˆˆˆ d� qd S )NT©Úrecord)Úcheck_namesÚ
check_likeÚcheck_dtype)rB   r   r   r   ÚtmÚassert_frame_equal)ÚrepeatÚ_Úactual©rc   rb   ra   rF   ÚexpectedÚpathÚread_kwargsÚwrite_kwargsr   r   ÚcompareÏ   s    .ûz!check_round_trip.<locals>.compare)rd   Úensure_clean)rF   r    rk   rm   rl   rj   ra   rb   rc   rf   rn   r   ri   r   Úcheck_round_trip£   s    "
(rp   c                 C   s|   t rLddlm} |j| dd�}t|jjƒt|ƒks6J ‚|jjt|ƒksxJ ‚n,ddlm	} |j	| dd�}|j
jj|ksxJ ‚dS )zÂCheck partitions of a parquet file are as expected.

    Parameters
    ----------
    path: str
        Path of the dataset.
    expected: iterable of str
        Expected partition names.
    r   NF)Zvalidate_schemaZhive)Úpartitioning)r   Úpyarrow.parquetÚparquetZParquetDatasetÚlenZ
partitionsZpartition_namesÚsetZpyarrow.datasetÚdatasetrq   ÚschemaÚnames)rk   rj   Úpqrv   Zdsr   r   r   Úcheck_partition_namesä   s    
rz   c                 C   sB   d}t jt|d�� t| ddƒ W d   ƒ n1 s40    Y  d S )Nz.engine must be one of 'pyarrow', 'fastparquet'©Úmatchr+   rM   )r"   ÚraisesÚ
ValueErrorrp   )r0   Úmsgr   r   r   Útest_invalid_engineû   s    r€   c                 C   s8   t  dd¡� t| ƒ W d   ƒ n1 s*0    Y  d S )Núio.parquet.enginer   ©r.   Úoption_contextrp   )r0   r$   r   r   r   Útest_options_py  s    r„   c                 C   s8   t  dd¡� t| ƒ W d   ƒ n1 s*0    Y  d S )Nr�   r   r‚   )r0   r&   r   r   r   Útest_options_fp  s    r…   c                 C   s8   t  dd¡� t| ƒ W d   ƒ n1 s*0    Y  d S )Nr�   Úautor‚   )r0   r&   r$   r   r   r   Útest_options_auto  s    r‡   c                 C   sV  t tdƒtƒsJ ‚t tdƒtƒs$J ‚t dd¡�F t tdƒtƒsDJ ‚t tdƒtƒsVJ ‚t tdƒtƒshJ ‚W d   ƒ n1 s|0    Y  t dd¡�F t tdƒtƒs¦J ‚t tdƒtƒs¸J ‚t tdƒtƒsÊJ ‚W d   ƒ n1 sÞ0    Y  t dd¡�L t tdƒtƒ�s
J ‚t tdƒtƒ�sJ ‚t tdƒtƒ�s2J ‚W d   ƒ n1 �sH0    Y  d S )Nr   r   r�   r†   )Ú
isinstancer   r   r
   r.   rƒ   )r&   r$   r   r   r   Útest_options_get_engine  s    00r‰   c                  C   s‚  ddl m}  |  d¡}|  d¡}ts(dnttjƒt|ƒk }tsBdnttjƒt|ƒk }to\| }tof| }|�s~|�s~|r¼d|› d�}t	j
t|d�� td	ƒ W d   ƒ qö1 s°0    Y  n:d
}t	j
t|d�� td	ƒ W d   ƒ n1 sì0    Y  |�rBd|› d�}t	j
t|d�� td	ƒ W d   ƒ n1 �s60    Y  n<d}t	j
t|d�� td	ƒ W d   ƒ n1 �st0    Y  d S )Nr   )ÚVERSIONSr   r   FzPandas requires version .z. or newer of .pyarrow.r{   r†   z%Missing optional dependency .pyarrow.z. or newer of .fastparquet.z)Missing optional dependency .fastparquet.)Zpandas.compat._optionalrŠ   Úgetr!   r	   r   Ú__version__r%   r   r"   r}   ÚImportErrorr   )rŠ   Z
pa_min_verZ
fp_min_verZhave_pa_bad_versionZhave_fp_bad_versionZhave_usable_paZhave_usable_fpr|   r   r   r   Ú"test_get_engine_auto_error_message*  s8    

ÿýÿý

(&*rŽ   c                 C   s~   | }t  ¡ �^}|j||d d� t||d�}t  ||¡ t||ddgd�}t  ||ddg ¡ W d   ƒ n1 sp0    Y  d S )N©r    r^   ©r    r<   r>   ©r    Úcolumns)rd   ro   r   r   re   )rG   r$   r&   rF   rk   Úresultr   r   r   Útest_cross_engine_pa_fpU  s    
r”   c              	   C   s¨   |}t  ¡ �ˆ}|j||d d� tdd��N t||d�}t  ||¡ t||ddgd�}t  ||ddg ¡ W d   ƒ n1 s|0    Y  W d   ƒ n1 sš0    Y  d S )Nr�   Tr_   r�   r<   r>   r‘   )rd   ro   r   r   r   re   )r   rG   r$   r&   rF   rk   r“   r   r   r   Útest_cross_engine_fp_pac  s    
r•   c                   @   s2   e Zd Zdd„ Zdd„ Zejjejdd„ ƒƒZ	dS )ÚBasec              	   C   sj   t  ¡ �N}tj||d��  t|||d d� W d   ƒ n1 s>0    Y  W d   ƒ n1 s\0    Y  d S )Nr{   ©r^   )rd   ro   r"   r}   r   )ÚselfrF   r    ÚexcÚerr_msgrk   r   r   r   Úcheck_error_on_writer  s    
zBase.check_error_on_writec              	   C   sf   t  ¡ �J}t  |¡�  t|||d d� W d   ƒ n1 s:0    Y  W d   ƒ n1 sX0    Y  d S )Nr—   )rd   ro   Zexternal_error_raisedr   )r˜   rF   r    r™   rk   r   r   r   Úcheck_external_error_on_writex  s    
z"Base.check_external_error_on_writec                 C   s.   |dkrt  |¡ d}t|ƒ}t ||¡ d S )Nr†   zdhttps://raw.githubusercontent.com/pandas-dev/pandas/main/pandas/tests/io/data/parquet/simple.parquet)r"   Úimportorskipr   rd   re   )r˜   r0   r    ÚurlrF   r   r   r   Útest_parquet_read_from_url~  s    
ÿzBase.test_parquet_read_from_urlN)
Ú__name__Ú
__module__Ú__qualname__r›   rœ   r"   ÚmarkÚnetworkrd   rŸ   r   r   r   r   r–   q  s
   r–   c                   @   s°   e Zd Zdd„ Zdd„ Zdd„ Zej dg d¢¡d	d
„ ƒZ	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ Zej d!g d"¢¡d#d$„ ƒZd%S )&Ú	TestBasicc                 C   sF   t  g d¢¡ddt  d¡t g d¢¡fD ]}d}|  ||t|¡ q(d S )Nr'   r(   r+   r9   z+to_parquet only supports IO with DataFrames)r.   ÚSeriesr[   rC   r   r›   r~   )r˜   r    Úobjr   r   r   r   Ú
test_errorŒ  s    ûzTestBasic.test_errorc                 C   s6   t  tdƒttddƒƒdœ¡}ddg|_t||ƒ d S )Nr1   r(   r2   ©rR   rU   r+   rM   )r.   r/   rA   rB   r’   rp   )r˜   r    rF   r   r   r   Útest_columns_dtypes—  s    
zTestBasic.test_columns_dtypesc                 C   s�   t  tdƒttddƒƒdœ¡}d}ddg|_|  ||t|¡ ddg|_|  ||t|¡ t d	dddd¡t d	dddd¡g|_|  ||t|¡ d S )
Nr1   r(   r2   r©   ú%parquet must have string column namesr   rJ   rK   iÛ  )r.   r/   rA   rB   r’   r›   r~   rX   )r˜   r    rF   r   r   r   r   Útest_columns_dtypes_invalidž  s    

þz%TestBasic.test_columns_dtypes_invalidr^   )NÚgzipÚsnappyÚbrotlic                 C   sN   |dkrt  d¡ n|dkr&t  d¡ t dg d¢i¡}t||d|id� d S )Nr®   r¯   r,   r'   r^   ©rm   )r"   r�   r.   r/   rp   )r˜   r    r^   rF   r   r   r   Útest_compression±  s    
zTestBasic.test_compressionc                 C   sJ   t  tdƒttddƒƒdœ¡}t  dtdƒi¡}t|||ddgid� d S )Nr1   r(   r2   r©   rR   r’   ©rj   rl   )r.   r/   rA   rB   rp   )r˜   r    rF   rj   r   r   r   Útest_read_columns½  s
    ÿzTestBasic.test_read_columnsc                 C   sž   |dk}t  dg d¢i¡}t||ƒ g d¢t jddd�tdƒg d	¢g}|D ]2}||_t|t jƒrn|j d ¡|_t|||d
� qJg d¢|_d|j_	t||ƒ d S )Nr   r,   r'   )r)   r*   r2   r9   r*   r:   r1   )r(   r*   r2   )ra   )r   r(   r)   r+   )
r.   r/   rp   rE   rA   Úindexrˆ   ZDatetimeIndexÚ
_with_freqÚname)r˜   r    ra   rF   Zindexesr´   r   r   r   Útest_write_indexÆ  s     
ü
zTestBasic.test_write_indexc                 C   s:   |}t  dg d¢i¡}t j g d¢¡}||_t||ƒ d S )Nr,   r'   ©)r<   r(   )r<   r)   )r=   r(   )r.   r/   Ú
MultiIndexÚfrom_tuplesr´   rp   )r˜   r$   r    rF   r´   r   r   r   Útest_write_multiindexÞ  s
    zTestBasic.test_write_multiindexc                 C   s¢   |}t jdddd�}t jtj dt|ƒ d¡tdƒd�}t jj	d	d
g|gddgd�}|j
d d�}||fD ]4}||_t||ƒ t||dddgi|ddg d� qhd S )Nz01-Jan-2018z01-Dec-2018ZMS)Úfreqr)   r*   ÚABC©r’   ÚLevel1ÚLevel2ÚlevelÚdate)rx   r’   r,   r-   )rl   rj   )r.   rE   r/   rC   ÚrandomÚrandnrt   rA   r¹   Zfrom_productÚcopyr´   rp   )r˜   r$   r    ÚdatesrF   Zindex1Úindex2r´   r   r   r   Útest_multiindex_with_columnsç  s    $ÿ
ÿz&TestBasic.test_multiindex_with_columnsc                 C   sÈ   t  g d¢g d¢dœ¡}d ddœ}|jdd�}t||||d� t jg d¢g d¢dœg d	¢d
�}t||||d� g d¢g d¢g}t jttdƒƒdd„ tdƒD ƒdœ|d
�}|jdd�}t||||d� d S )Nr'   )ÚqÚrÚs)r<   r=   F)r^   r´   T)Údrop©rm   rj   )ZzyxZwvuZtsr)r´   ©rM   rM   rN   rN   r+   r+   ÚquxrÏ   ©ÚoneÚtworÑ   rÒ   rÑ   rÒ   rÑ   rÒ   é   c                 S   s   g | ]
}| ‘qS r   r   )Ú.0Úir   r   r   Ú
<listcomp>  ó    z7TestBasic.test_write_ignoring_index.<locals>.<listcomp>)rÑ   rÒ   )r.   r/   Zreset_indexrp   rA   rB   )r˜   r    rF   rm   rj   Úarraysr   r   r   Útest_write_ignoring_index÷  s     
ÿþ ÿz#TestBasic.test_write_ignoring_indexc                 C   s@   t j g d¢¡}t jtj dd¡|d�}d}|  ||t|¡ d S )Nr¸   r2   r*   r¾   úY\s*parquet must have string column names for all values in\s*each level of the MultiIndex)	r.   r¹   rº   r/   rC   rÃ   rÄ   r›   r~   )r˜   r    Z
mi_columnsrF   r   r   r   r   Útest_write_column_multiindex  s
    ÿz&TestBasic.test_write_column_multiindexc                 C   sP   |}g d¢g d¢g}t jtj dd¡|d�}ddg|j_d}|  ||t|¡ d S )NrÎ   )r(   r)   r(   r)   r(   r)   r(   r)   rÓ   r¾   r¿   rÀ   rÚ   )	r.   r/   rC   rÃ   rÄ   r’   rx   r›   r~   ©r˜   r$   r    rØ   rF   r   r   r   r   Ú&test_write_column_multiindex_nonstring!  s    þÿz0TestBasic.test_write_column_multiindex_nonstringc                 C   sF   |}g d¢g d¢g}t jtj dd¡|d�}ddg|j_t||ƒ d S )NrÎ   rÐ   rÓ   r¾   Z	ColLevel1Z	ColLevel2)r.   r/   rC   rÃ   rÄ   r’   rx   rp   ©r˜   r$   r    rØ   rF   r   r   r   Ú#test_write_column_multiindex_string3  s    þz-TestBasic.test_write_column_multiindex_stringc                 C   s:   |}g d¢}t jtj dd¡|d�}d|j_t||ƒ d S )N)rM   rN   r+   rÏ   rÓ   r2   r¾   Z	StringCol)r.   r/   rC   rÃ   rÄ   r’   r¶   rp   rÞ   r   r   r   Útest_write_column_index_stringB  s
    z(TestBasic.test_write_column_index_stringc                 C   sD   |}g d¢}t jtj dd¡|d�}d|j_d}|  ||t|¡ d S )N©r(   r)   r*   r2   rÓ   r2   r¾   ZNonStringColr«   )	r.   r/   rC   rÃ   rÄ   r’   r¶   r›   r~   rÜ   r   r   r   Ú!test_write_column_index_nonstringN  s    z+TestBasic.test_write_column_index_nonstringc           
   
   C   st  dd l m} |dkr.tjjdd�}|j |¡ t t 	g d¢d¡t 	g d¢d¡t 	g d¢¡t 	g d	¢¡t 	g d
¢d¡dœ¡}t
 ¡ �6}| ||¡ t||d�}t||dd�}W d   ƒ n1 sÀ0    Y  |d jt d¡ksâJ ‚t tj	g d¢dd�tj	g d¢dd�tj	g d¢dd�tj	g d	¢dd�tj	g d
¢dd�dœ¡}	|dk�rd|jddd�}|	jddd�}	t
 ||	¡ d S )Nr   r   z.Fastparquet nullable dtype support is disabledr   ©r(   r)   r*   NZint64Zuint8)r<   r=   rH   N)TFTNrá   )r<   r=   rH   r>   r?   r�   T©r    Úuse_nullable_dtypesr<   r5   ÚInt64r6   ÚUInt8rR   ÚbooleanrH   r(   )Zaxis)rr   rs   r"   r£   ZxfailÚnodeZ
add_markerr   Útabler   rd   ro   Zwrite_tabler   r7   rC   r.   r/   rÌ   re   )
r˜   r    r   ry   r£   rê   rk   Zresult1Zresult2rj   r   r   r   Útest_use_nullable_dtypesZ  s>    ÿúÿ

,ûÿ	
z"TestBasic.test_use_nullable_dtypesr7   )	ræ   rç   rè   Úobjectzdatetime64[ns, UTC]rV   z	period[D]ZFloat64rR   c                 C   s.   t  dt jg |d�i¡}t||ddid� d S )NÚvaluer6   rå   T)rl   )r.   r/   r   rp   )r˜   r$   r7   rF   r   r   r   Útest_read_empty_array†  s
    ÿÿzTestBasic.test_read_empty_arrayN)r    r¡   r¢   r¨   rª   r¬   r"   r£   Úparametrizer±   r³   r·   r»   rÈ   rÙ   rÛ   rÝ   rß   rà   râ   rë   rî   r   r   r   r   r¥   ‹  s*   
		 
,þr¥   z8ignore:CategoricalBlock is deprecated:DeprecationWarningc                   @   s:  e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
e d¡ej ddgg g¡dd„ ƒƒZe d¡dd„ ƒZe d¡dd„ ƒZdd„ Zdd„ Zej deejg¡d d!„ ƒZd"d#„ Zd$d%„ Ze d¡d&d'„ ƒZejdd(d)�d*d+„ ƒZe d¡d,d-„ ƒZd.d/„ Zd0d1„ Zejdd(d)�d2d3„ ƒZ d4d5„ Z!d6S )7ÚTestParquetPyArrowc                 C   s@   |}t jdddd�}| d ¡}||d< g d¢|d< t||ƒ d S )Nr9   r*   úEurope/Brussels©r;   ÚtzÚdatetime_tz)TNTZbool_with_none)r.   rE   rµ   rp   )r˜   r$   r\   rF   Údtir   r   r   Ú
test_basic   s    
zTestParquetPyArrow.test_basicc                 C   s<   |}t jdddd�|d< t|||ddg dddgid	� d S )
Nr9   r*   rñ   rò   rô   rR   rU   r’   r²   )r.   rE   rp   )r˜   r$   r\   rF   r   r   r   Útest_basic_subset_columns¬  s    

üz,TestParquetPyArrow.test_basic_subset_columnsc                 C   s:   |j |d�}t|tƒsJ ‚t|ƒ}t|ƒ}t ||¡ d S )Nr�   )r   rˆ   rS   r   r   rd   re   )r˜   r$   r\   Z	buf_bytesZ
buf_streamÚresr   r   r   Ú*test_to_bytes_without_path_or_buf_providedº  s
    z=TestParquetPyArrow.test_to_bytes_without_path_or_buf_providedc                 C   s8   t jt d¡ dd¡tdƒd� ¡ }|  ||td¡ d S )Né   r2   r*   Úaaar¾   zDuplicate column names found©	r.   r/   rC   rD   ZreshaperA   rÅ   r›   r~   ©r˜   r$   rF   r   r   r   Útest_duplicate_columnsÅ  s    $z)TestParquetPyArrow.test_duplicate_columnsc                 C   sL   t  dt jddd�i¡}|  ||t¡ t  dg d¢i¡}|  ||tj¡ d S )Nr<   ú1 dayr*   r:   ©r<   r(   rQ   )r.   r/   Útimedelta_rangerœ   ÚNotImplementedErrorr   ZArrowExceptionrý   r   r   r   Útest_unsupportedÊ  s    z#TestParquetPyArrow.test_unsupportedc                 C   sd   t  ¡ }t  tdƒ¡|d< t jg d¢t  g d¢¡d�|d< t jg d¢g d¢d	d
�|d< t||ƒ d S )NZabcdefr<   )rM   r+   r+   rM   NrM   rL   r6   r=   )r<   r=   rH   r<   rH   r=   )r=   rH   r>   T)Ú
categoriesZorderedrH   )r.   r/   ÚCategoricalrA   ZCategoricalDtyperp   rý   r   r   r   Útest_categoricalÕ  s    þ
ÿ
z#TestParquetPyArrow.test_categoricalc                 C   s8   t  d¡}|jf i |¤Ž}d|i}t||d||d� d S )NÚs3fsÚ
filesystemzpandas-test/pyarrow.parquet©rk   rl   rm   )r"   r�   ZS3FileSystemrp   )r˜   r0   Ús3_resourcer$   Ús3sor  Zs3Úkwr   r   r   Útest_s3_roundtrip_explicit_fsè  s    
ûz0TestParquetPyArrow.test_s3_roundtrip_explicit_fsc                 C   s   d|i}t ||d||d� d S )NÚstorage_optionsz s3://pandas-test/pyarrow.parquetr	  ©rp   )r˜   r0   r
  r$   r  r   r   r   Útest_s3_roundtripô  s    ûz$TestParquetPyArrow.test_s3_roundtripr  Úpartition_colr,   c              
   C   sR   |  ¡ }|r*trdnd}||  |¡||< t|||dd|i|d |dœddd� d S )	NZint32r   zs3://pandas-test/parquet_dirr  )Úpartition_colsr^   r  Tr(   )rj   rk   rl   rm   rb   rf   )rÅ   r   rZ   rp   )r˜   r0   r
  r$   r  r  Zexpected_dfZpartition_col_typer   r   r   Útest_s3_roundtrip_for_dirÿ  s&    ÿýôz,TestParquetPyArrow.test_s3_roundtrip_for_dirr   c                 C   s(   t ƒ }| |¡ t|ƒ}t ||¡ d S r   )r   r   r   rd   re   )r˜   r0   ÚbufferZdf_from_bufr   r   r   Útest_read_file_like_obj_support)  s    
z2TestParquetPyArrow.test_read_file_like_obj_supportc                 C   sŠ   |  dd¡ |  dd¡ tjtdd�� tdƒ W d   ƒ n1 sD0    Y  tjtdd�� | d¡ W d   ƒ n1 s|0    Y  d S )NÚHOMEZTestingUserÚUSERPROFILEz.*TestingUser.*r{   z~/file.parquet)Zsetenvr"   r}   ÚOSErrorr   r   )r˜   r0   Úmonkeypatchr   r   r   Útest_expand_user0  s    &z#TestParquetPyArrow.test_expand_userc                 C   sf   ddg}|}t  ¡ �>}|j||d d� t||ƒ t|ƒj|jksDJ ‚W d   ƒ n1 sX0    Y  d S )NrW   rU   ©r  r^   ©rd   Úensure_clean_dirr   rz   r   Úshape)r˜   r$   r\   r  rF   rk   r   r   r   Útest_partition_cols_supported9  s    

z0TestParquetPyArrow.test_partition_cols_supportedc                 C   sh   d}|g}|}t  ¡ �>}|j||d d� t||ƒ t|ƒj|jksFJ ‚W d   ƒ n1 sZ0    Y  d S )NrW   r  r  )r˜   r$   r\   r  Úpartition_cols_listrF   rk   r   r   r   Útest_partition_cols_stringB  s    

z-TestParquetPyArrow.test_partition_cols_stringÚ	path_typec           	      C   sd   d}|g}|}t  ¡ �:}||ƒ}|j||d� t|ƒj|jksBJ ‚W d   ƒ n1 sV0    Y  d S )Nr-   )r  )rd   r  r   r   r  )	r˜   r$   r0   r"  r  r   rF   Zpath_strrk   r   r   r   Útest_partition_cols_pathlibL  s    
z.TestParquetPyArrow.test_partition_cols_pathlibc                 C   s   t  ¡ }t||ƒ d S r   )r.   r/   rp   rý   r   r   r   Útest_empty_dataframeY  s    z'TestParquetPyArrow.test_empty_dataframec                 C   sV   dd l }t dddgi¡}| |jd| ¡ d�g¡}| t¡}t||d|i|d� d S )Nr   Úxr(   )Útyperw   rÍ   )	r   r.   r/   rw   ÚfieldZbool_rZ   rW   rp   )r˜   r$   r   rF   rw   Zout_dfr   r   r   Útest_write_with_schema^  s
    
z)TestParquetPyArrow.test_write_with_schemac                 C   sp   t  t jg d¢dd�t jg d¢dd�t jg d¢dd�dœ¡}t||ƒ t  dt jg d	¢dd�i¡}t||ƒ d S )
Nr'   ræ   r6   ZUInt32rI   rR   )r<   r=   rH   r<   rã   )r.   r/   r¦   rp   rý   r   r   r   Ú test_additional_extension_arraysf  s    ýÿ
z3TestParquetPyArrow.test_additional_extension_arraysz1.0.0)Úmin_versionc              	   C   sh   t  dt jg d¢dd�i¡}t  d|¡�, t||| d|› d�¡d� W d   ƒ n1 sZ0    Y  d S )	Nr<   rI   zstring[pyarrow]r6   Ústring_storagezstring[ú]©rj   )r.   r/   r¦   rƒ   rp   rZ   )r˜   r$   r+  rF   r   r   r   Ú test_pyarrow_backed_string_arrayv  s    z3TestParquetPyArrow.test_pyarrow_backed_string_arrayc                 C   s(   t  dt jdddd�i¡}t||ƒ d S )Nr>   z
2012-01-01r*   ÚD)r;   r¼   )r.   r/   Úperiod_rangerp   rý   r   r   r   Útest_additional_extension_types}  s
    ýÿz2TestParquetPyArrow.test_additional_extension_typesc                 C   s>   t s
d}nd}t dtjdddd�i¡}t||d|id	� d S )
Nz2.6z2.0r<   z
2017-01-01Z1né
   ©r¼   r;   Úversionr°   )r   r.   r/   rE   rp   )r˜   r$   ÚverrF   r   r   r   Útest_timestamp_nanosecondsŠ  s
    z-TestParquetPyArrow.test_timestamp_nanosecondsc                 C   s:   t st ¡  d|g }tj|d|id�}t||dd� d S )Né   Úindex_as_col©r´   ÚdataF)rc   )r   r"   r#   r.   r/   rp   )r˜   r$   r]   ÚidxrF   r   r   r   Útest_timezone_aware_index•  s
    
z,TestParquetPyArrow.test_timezone_aware_indexc                 C   sr   t  dttddƒƒi¡}t ¡ �.}| ||¡ t||dgdd�}W d   ƒ n1 sT0    Y  t|ƒdksnJ ‚d S )Nr<   r   r*   ©r<   z==r   F)ÚfiltersZuse_legacy_datasetr(   )	r.   r/   rA   rB   rd   ro   r   r   rt   )r˜   r$   rF   rk   r“   r   r   r   Útest_filter_row_groups¨  s    

ÿ$z)TestParquetPyArrow.test_filter_row_groupsc                 C   s�   t jtj dd¡g d¢d�}t ¡ �&}| ||¡ t||ƒ}W d   ƒ n1 sP0    Y  |rvt	|j
t jjjƒsŒJ ‚nt	|j
t jjjƒsŒJ ‚d S )Nr2  r*   )r,   r-   ÚCr¾   )r.   r/   rC   rÃ   rÄ   rd   ro   r   r   rˆ   Z_mgrÚcoreZ	internalsZArrayManagerZBlockManager)r˜   r$   Zusing_array_managerrF   rk   r“   r   r   r   Útest_read_parquet_manager³  s    
(z,TestParquetPyArrow.test_read_parquet_managerN)"r    r¡   r¢   rö   r÷   rù   rþ   r  r  r  r  ÚtdZ
skip_if_nor"   r£   rï   r  r  r  r  r!  ÚstrÚpathlibÚPathr#  r$  r(  r)  r.  r1  r6  r<  r?  rB  r   r   r   r   rð   ž  sJ   þþ"

	






rð   c                   @   sŠ   e Zd Zdd„ Zejjdd�dd„ ƒZdd„ Zd	d
„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ ZdS ) ÚTestParquetFastParquetc                 C   sF   |}t jdddd�}| d ¡}||d< t jddd�|d< t||ƒ d S )	Nr9   r*   z
US/Easternrò   rô   rÿ   r:   Ú	timedelta)r.   rE   rµ   r  rp   )r˜   r&   r\   rF   rõ   r   r   r   rö   Á  s    
z!TestParquetFastParquet.test_basicznot supportedr   c                 C   s<   t jt d¡ dd¡tdƒd� ¡ }d}|  ||t|¡ d S )Nrú   r2   r*   rû   r¾   z9Cannot create parquet dataset with duplicate column namesrü   ©r˜   r&   rF   r   r   r   r   rþ   Ê  s    $z-TestParquetFastParquet.test_duplicate_columnsc                 C   s@   t  dg d¢i¡}t jddtjdgidd�}t|||dd� d S )	Nr<   )TNFg      ð?g        Zfloat16r6   F)rj   rc   )r.   r/   rC   rY   rp   ©r˜   r&   rF   rj   r   r   r   Útest_bool_with_noneÒ  s    z*TestParquetFastParquet.test_bool_with_nonec                 C   sT   t  dt jdddd�i¡}|  ||td ¡ t  dg d¢i¡}d}|  ||t|¡ d S )Nr<   Z2013ÚMr*   r3  r   z"Can't infer object conversion type)r.   r/   r0  r›   r~   rI  r   r   r   r  Ù  s
    z'TestParquetFastParquet.test_unsupportedc                 C   s&   t  dt  tdƒ¡i¡}t||ƒ d S )Nr<   r1   )r.   r/   r  rA   rp   )r˜   r&   rF   r   r   r   r  å  s    z'TestParquetFastParquet.test_categoricalc                 C   sz   dt tddƒƒi}t |¡}t ¡ �2}|j||d dd� t||dgd�}W d   ƒ n1 s\0    Y  t|ƒdksvJ ‚d S )Nr<   r   r*   r(   )r^   Zrow_group_offsetsr=  )r>  )	rA   rB   r.   r/   rd   ro   r   r   rt   )r˜   r&   r>   rF   rk   r“   r   r   r   r?  é  s    

.z-TestParquetFastParquet.test_filter_row_groupsc                 C   s    t ||dd|id |dœd� d S )Nz$s3://pandas-test/fastparquet.parquetr  )r^   r  r	  r  )r˜   r0   r
  r&   r  r   r   r   r  ñ  s    ûz(TestParquetFastParquet.test_s3_roundtripc                 C   s€   ddg}|}t  ¡ �X}|j|d|d d� tj |¡s8J ‚dd l}| |d¡j}t	|ƒdks^J ‚W d   ƒ n1 sr0    Y  d S )NrW   rU   r   ©r    r  r^   r   Fr)   ©
rd   r  r   Úosrk   Úexistsr   ZParquetFileZcatsrt   ©r˜   r&   r\   r  rF   rk   r   Zactual_partition_colsr   r   r   r  û  s    
üz4TestParquetFastParquet.test_partition_cols_supportedc                 C   s|   d}|}t  ¡ �X}|j|d|d d� tj |¡s4J ‚dd l}| |d¡j}t	|ƒdksZJ ‚W d   ƒ n1 sn0    Y  d S )NrW   r   rM  r   Fr(   rN  rQ  r   r   r   r!    s    
üz1TestParquetFastParquet.test_partition_cols_stringc                 C   s€   ddg}|}t  ¡ �X}|j|dd |d� tj |¡s8J ‚dd l}| |d¡j}t	|ƒdks^J ‚W d   ƒ n1 sr0    Y  d S )NrW   rU   r   )r    r^   Úpartition_onr   Fr)   rN  rQ  r   r   r   Útest_partition_on_supported  s    
üz2TestParquetFastParquet.test_partition_on_supportedc              	   C   s~   ddg}|}d}t jt|d��L t ¡ �$}|j|dd ||d� W d   ƒ n1 sR0    Y  W d   ƒ n1 sp0    Y  d S )NrW   rU   zYCannot use both partition_on and partition_cols. Use partition_cols for partitioning datar{   r   )r    r^   rR  r  )r"   r}   r~   rd   r  r   )r˜   r&   r\   r  rF   r   rk   r   r   r   Ú3test_error_on_using_partition_cols_and_partition_on.  s    ÿ
ûzJTestParquetFastParquet.test_error_on_using_partition_cols_and_partition_onc                 C   s*   t  ¡ }| ¡ }d|j_t|||d� d S )Nr´   r-  ©r.   r/   rÅ   r´   r¶   rp   rJ  r   r   r   r$  @  s    z+TestParquetFastParquet.test_empty_dataframec                 C   s>   d|g }t j|d|id�}| ¡ }d|j_t|||d� d S )Nr7  r8  r9  r´   r-  rU  )r˜   r&   r]   r;  rF   rj   r   r   r   r<  G  s
    
z0TestParquetFastParquet.test_timezone_aware_indexc              	   C   s„   t  dddgi¡}t ¡ �V}| |¡ tjtdd�� t|ddd� W d   ƒ n1 sX0    Y  W d   ƒ n1 sv0    Y  d S )	Nr<   r(   r)   z!not supported for the fastparquetr{   r   Trä   )	r.   r/   rd   ro   r   r"   r}   r~   r   )r˜   r  r&   rF   rk   r   r   r   Ú&test_use_nullable_dtypes_not_supportedP  s
    

z=TestParquetFastParquet.test_use_nullable_dtypes_not_supportedN)r    r¡   r¢   rö   r"   r£   r#   rþ   rK  r  r  r?  r  r  r!  rS  rT  r$  r<  rV  r   r   r   r   rG  À  s   	

	rG  )	NNNNNTFTr)   )JÚ__doc__rX   Úior   rO  rE  Úwarningsr   r   ÚnumpyrC   r"   Zpandas._configr   Zpandas.compat.pyarrowr   r   r   Zpandas.util._test_decoratorsÚutilZ_test_decoratorsrC  Zpandasr.   Zpandas._testingZ_testingrd   Zpandas.util.versionr	   Zpandas.io.parquetr
   r   r   r   r   r   r!   r�   ÚDeprecationWarningÚFutureWarningr   r%   r£   Z
pytestmarkZfixturer   Zskipifr    r$   r&   r0   rG   r\   ÚnowÚtimezoneÚutcÚminÚmaxÚstrptimer]   rp   rz   r€   r„   r…   r‡   r‰   rŽ   r”   r•   r–   r¥   rð   rG  r   r   r   r   Ú<module>   sÀ   
(
ÿþþÿþøÿ





ùÿ
         ö
A+  
  #