a
    nÝEb>%  ã                   @   s  d Z ddlmZ ddlZddlZddlZddlZddlm	Z	m
Z
 ddlmZ ej d¡Zedd„ ƒZedd	„ ƒZeej d
ddg¡ej dg d¢¡dd„ ƒƒƒZedd„ ƒZedd„ ƒZeej ddi e	ddgiƒfdddie	ddgiƒfdddgie	dddgiƒfd dgd!d"œe	ddgiƒfd dgd#d"œe	dejdgiƒfg¡d$d%„ ƒƒZed&d'„ ƒZeej d(g d)¢¡d*d+„ ƒƒZeej d,d!d#g¡d-d.„ ƒƒZed/d0„ ƒZej dg d1¢¡d2d3„ ƒZeej dg d4¢¡d5d6„ ƒƒZed7d8„ ƒZed9d:„ ƒZ ej d;¡ej d<d=d>g¡d?d@„ ƒƒZ!dS )AzZ
Tests encoding functionality during parsing
for all of the parsers defined in parsers.py
é    )ÚBytesION)Ú	DataFrameÚread_csvZpyarrow_skipc                 C   sL   d}| }t d |¡ƒ}|j|d|d�}tddggddgd	�}t ||¡ d S )
NÚcp1255u   ×©×œ×•×�:1234
562:123ú:)ÚsepÚencodingi2  é{   u   ×©×œ×•×�Ú1234©Úcolumns©r   Úencoder   r   ÚtmÚassert_frame_equal)Úall_parsersr   ÚparserÚdataÚresultÚexpected© r   úd/home/ja/django-apps/lartica_env/lib/python3.9/site-packages/pandas/tests/io/parser/test_encoding.pyÚtest_bytes_io_input   s    r   c                 C   s@   | }t d ¡ ƒ}|j|ddd d�}tddggƒ}t ||¡ d S )Nu   Å�aski, Jan;1ú;úutf-8)r   r   Úheaderu   Å�aski, Jané   r   )r   r   r   r   r   r   r   r   Útest_read_csv_unicode"   s
    r   r   ú,ú	r   )úutf-16zutf-16lezutf-16bec              	   C   s  | }d  d|¡}dt d¡› d�}|ddœ}d}t |¡�´}d	d
lm} | |¡}	t|dƒ�}
|
 |	¡ W d   ƒ n1 s|0    Y  t	| |¡ƒ}|||d�}|j
|fd|i|¤Ž}|j
|fd|i|¤Ž}| ¡  t ||¡ W d   ƒ n1 sô0    Y  d S )Nz)skip this
skip this too
A,B,C
1,2,3
4,5,6r   Ú__é
   z__.csvé   )r   Zskiprowsr   r   )ÚTextIOWrapperÚwb©r   r   )Úreplacer   ZrandsÚensure_cleanÚior$   r   ÚopenÚwriter   r   Úcloser   )r   r   r   r   r   ÚpathÚkwargsÚutf8r$   Ú
bytes_dataÚfZbytes_bufferr   r   r   r   r   Útest_utf16_bom_skiprows,   s$    û

(r2   c                 C   s6   t j |d¡}| }|j|ddd�}t|ƒdks2J ‚d S )Nzutf16_ex.txtr    r   )r   r   é2   )Úosr-   Újoinr   Úlen)r   Úcsv_dir_pathr-   r   r   r   r   r   Útest_utf16_exampleO   s    r8   c                 C   sL   t j |d¡}| }|j|d dd�}| d¡}|d d }d}||ksHJ ‚d S )Núunicode_series.csvúlatin-1)r   r   r   r   i`  u$   Ã� kÃ¶ldum klaka (Cold Fever) (1994))r4   r-   r5   r   Z	set_index)r   r7   r-   r   r   Úgotr   r   r   r   Útest_unicode_encodingW   s    
r<   zdata,kwargs,expectedza
1Úar   z"a"
1Ú	quotecharú"zb
1ÚnamesÚbÚ1z
1T)r@   Zskip_blank_linesFc                    sD   | }d‰ d‰‡ ‡fdd„}|j ||ƒfdˆi|¤Ž}t ||¡ d S )Nu   ï»¿r   c                    s   ˆ |    ˆ¡}t|ƒS )N)r   r   )Ú_dataZbom_data©Úbomr/   r   r   Ú_encode_data_with_bom~   s    z,test_utf8_bom.<locals>._encode_data_with_bomr   )r   r   r   )r   r   r.   r   r   rF   r   r   rD   r   Útest_utf8_bomd   s    rG   c                 C   sL   t dgdgdœƒ}| }| |¡}d |¡}|jt|ƒ|d�}t ||¡ d S )Ng333333@Útest)Zmb_numZ	multibytezmb_num,multibyte
4.8,testr&   )r   Úformatr   r   r   r   r   )r   Ú	utf_valueÚencoding_fmtr   r   r   r   r   r   r   r   Útest_read_csv_utf_aliases†   s    

rL   zfile_path,encoding)))r)   r   Úcsvz	test1.csvr   ))r)   r   r   r9   r:   ))r)   r   r   zsauron.SHIFT_JIS.csvÚshiftjisc                 C   s  | }||Ž }|j ||d�}t||d��$}|  |¡}	|jr<J ‚W d   ƒ n1 sP0    Y  t ||	¡ t|dd��(}
|j |
|d�}	|
jrŒJ ‚W d   ƒ n1 s 0    Y  t ||	¡ t|ddd��(}
|j |
|d�}	|
jrÞJ ‚W d   ƒ n1 sò0    Y  t ||	¡ d S )Nr&   Úrb©Úmoder   )rQ   Ú	buffering)r   r*   Úclosedr   r   )r   r7   Ú	file_pathr   Údatapathr   Zfpathr   Úfar   Zfbr   r   r   Útest_binary_mode_file_buffers“   s    
(((rW   Úpass_encodingc           	      C   s†   | }|  |¡}tddgiƒ}tjd|dd��F}| d¡ | d¡ |j||rP|nd d�}t ||¡ W d   ƒ n1 sx0    Y  d S )	NZfooÚbarzw+T)rQ   r   Zreturn_filelikezfoo
barr   r&   )rI   r   r   r(   r+   Úseekr   r   )	r   rJ   rK   rX   r   r   r   r1   r   r   r   r   Útest_encoding_temp_file¶   s    


r[   c                 C   s’   | }d}d}d}t ||giƒ}t ¡ �X}| |› d|› � |¡¡ | d¡ |j||d�}t ||¡ |j	rpJ ‚W d   ƒ n1 s„0    Y  d S )Nz	shift-jisu	   ã�¦ã�™ã�¨u   ã�“ã‚€Ú
r   r&   )
r   ÚtempfileÚNamedTemporaryFiler+   r   rZ   r   r   r   rS   )r   r   r   Útitler   r   r1   r   r   r   r   Útest_encoding_named_temp_fileÇ   s    

r`   )r   r    z	utf-16-bez	utf-16-lezutf-32c                 C   sR   d}t | | ¡ƒ}t|d| d�}tddgddgdd	ggd
dgd�}t ||¡ d S )Nu   a	b
ï¼šfoo	0
bar	1
baz	2r   )Ú	delimiterr   u   ï¼šfoor   rY   r   Zbazr#   r=   rA   )r   r   r   )r   r   Zencoded_datar   r   r   r   r   Ú%test_parse_encoded_special_charactersÜ   s
    "rb   )r   Nr    r   r:   c                 C   sx   | }t g d¢g d¢g d¢dœƒ}t ¡ �0}|j|d|d� |j||dd�}W d   ƒ n1 s^0    Y  t ||¡ d S )	N)ZRaphaelZ	DonatellozMiguel AngelZLeonardo)ÚredÚpurpleZorangeÚblue)Zsaizbo staffZnunchunkZkatana)ÚnameÚmaskZweaponF)Úindexr   T)r   Ú
memory_map)r   r   r(   Úto_csvr   r   )r   r   r   r   ÚfileÚdfr   r   r   Útest_encoding_memory_mapê   s    ýÿ
.rm   c                 C   s|   | }t dgd d�}d|jd< t d¡�4}|j|dddd	� |j|d
ddd�}W d
  ƒ n1 sb0    Y  t ||¡ d
S )zO
    Chunk splits a multibyte character with memory_map=True

    GH 43540
    Zaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaai   )r   u�   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaÄ…iÿ  zbug-gh43540.csvFr   ©rh   r   r   NTÚc)r   ri   Úengine)r   Zilocr   r(   rj   r   r   )r   r   rl   ÚfnameÚdfrr   r   r   Ú test_chunk_splits_multibyte_charü   s    
0rs   c              	   C   sè   g }d}d}d}t t|ƒt|ƒ|ƒD ]V}d dd„ t ||d ƒD ƒ¡d }z| d¡ W n tyn   Y q$Y n0 | |¡ q$| }t|ƒ}t d	¡�6}	|j	|	d
d
dd� |j
|	ddddd�}
W d  ƒ n1 sÎ0    Y  t ||
¡ dS )zg
    GH 43787

    Test correct handling of UTF-8 chars when memory_map=True and encoding is UTF-8
    é€   ú u   ð�‚€Ú c                 S   s   g | ]}t |ƒ‘qS r   )Úchr)Ú.0ro   r   r   r   Ú
<listcomp>  ó    z,test_readcsv_memmap_utf8.<locals>.<listcomp>r\   r   zutf8test.csvFrn   NTro   )r   ri   rp   r   )ÚrangeÚordr5   r   ÚUnicodeEncodeErrorÚappendr   r   r(   rj   r   r   )r   ÚlinesÚline_lengthZ
start_charZend_charÚlnumÚliner   rl   rq   rr   r   r   r   Útest_readcsv_memmap_utf8  s&    "

ÿ$rƒ   Zpyarrow_xfailrQ   zw+bzw+tc                 C   s|   | }d}d|v rd}t j|d��.}| |¡ | d¡ | |¡}W d   ƒ n1 sT0    Y  tg dgd�}t ||¡ d S )Ns   abcdÚtÚabcdrP   r   r   )r]   ÚSpooledTemporaryFiler+   rZ   r   r   r   r   )r   rQ   r   ÚcontentÚhandlerl   r   r   r   r   Útest_not_readable.  s    

(r‰   )"Ú__doc__r)   r   r4   r]   ÚnumpyÚnpZpytestZpandasr   r   Zpandas._testingZ_testingr   ÚmarkZusefixturesZskip_pyarrowr   r   Zparametrizer2   r8   r<   ÚnanrG   rL   rW   r[   r`   rb   rm   rs   rƒ   r‰   r   r   r   r   Ú<module>   sz   

	 


ýöþ
þ
ÿ



