a oEb@sdZddlmZddlmZddlZddlmZmZddl Z ddl Z ddl Z ddl m Z ddlmZmZmZmZmZmZmZddlZddlZddlmZmZdd lmZmZdd l m!Z!dd l"m#Z#m$Z$m%Z%dd l&m'Z'dd l(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:ddl;mZ>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFddlGmHZHddlImJZJmKZKmLZLddlMmNmOZPddlQmRZRmSZSddlTmUZUddlVmWZWddlXmYZYmZZZddl[m\Z\ddl]m^Z^m_Z_erddl`maZambZbmcZcddlXmdZddZedZfd d!Zgd"d#Zhd$d%ZieRZjd&d'd(d)ZkGd*d+d+elZmGd,d-d-elZnGd.d/d/eoZpd0ZqGd1d2d2eoZrd3ZsGd4d5d5eoZtd6Zud7Zvd8d8d9d9d:Zwe>dgiZxd;Zydd?eyej}d@ej|dAdeze~gdBd@Wdn1s 0Ydad?adCdDZddHdIdHdJdKdLdKdLdMdNdOdHdHdPdQdRdSZddHdHdJdJdJdUdVdWZdXdXdLdYdZd[ZGd\d]d]ZGd^d_d_ZGd`dadaZGdbdcdceZGdddedeeZGdfdgdgeZGdhdidieZGdjdkdkZGdldmdmeZGdndodoeZGdpdqdqeZGdrdsdseZGdtdudueZGdvdwdweZGdxdydyeZGdzd{d{eZGd|d}d}eZGd~ddeZGdddeZGdddeZddd&dddddZdddddZddddLddddZdHddHdHdadddZdHdHdHddddZdHdddddZddHdHddddZddHdHddddZddHdHdHdddZdHdHdHdddZdHdLdddZdHddHdddZdHdHdddZddddZGdddZdS)zY High level interface to PyTables for reading and writing pandas data structures to disk ) annotations)suppressN)datetzinfo)dedent) TYPE_CHECKINGAnyCallableHashableLiteralSequencecast)config get_option)libwriters) timezones) ArrayLikeDtypeArgShape)import_optional_dependency) patch_pickle)PerformanceWarning)cache_readonly)find_stack_level) ensure_objectis_categorical_dtypeis_complex_dtypeis_datetime64_dtypeis_datetime64tz_dtypeis_extension_array_dtype is_list_likeis_string_dtypeis_timedelta64_dtypeneeds_i8_conversion)array_equivalent) DataFrame DatetimeIndexIndex MultiIndex PeriodIndexSeriesTimedeltaIndexconcatisna) Int64Index) Categorical DatetimeArray PeriodArray) PyTablesExprmaybe_expression) extract_array) ensure_index) ArrayManager BlockManager)stringify_path)adjoin pprint_thing)ColFileNode)Blockz0.15.2UTF-8cCst|tjr|d}|S)z(if we have bytes, decode them to unicoder@) isinstancenpbytes_decode)srFR/home/ja/django-apps/lartica_env/lib/python3.9/site-packages/pandas/io/pytables.py_ensure_decodedus  rHcCs|dur t}|SN)_default_encodingencodingrFrFrG_ensure_encoding|srMcCst|trt|}|S)z Ensure that an index / column name is a str (python 3); otherwise they may be np.string dtype. Non-string dtypes are passed through unchanged. https://github.com/pandas-dev/pandas/issues/13492 )rAstrnamerFrFrG _ensure_strs rQint scope_levelcsV|dt|ttfr*fdd|D}nt|r>t|d}|dusNt|rR|SdS)z Ensure that the where is a Term or a list of Term. This makes sure that we are capturing the scope of variables that are passed create the terms here with a frame_level=2 (we are 2 levels down) cs0g|](}|durt|r(t|ddn|qS)NrUrS)r4Term).0termlevelrFrG sz _ensure_term..rSN)rAlisttupler4rVlen)whererTrFrYrG _ensure_terms   r`c@s eZdZdS)PossibleDataLossErrorN__name__ __module__ __qualname__rFrFrFrGrasrac@s eZdZdS)ClosedFileErrorNrbrFrFrFrGrfsrfc@s eZdZdS)IncompatibilityWarningNrbrFrFrFrGrgsrgz where criteria is being ignored as this version [%s] is too old (or not-defined), read the file in and write it out to a new file to upgrade (with the copy_to method) c@s eZdZdS)AttributeConflictWarningNrbrFrFrFrGrhsrhzu the [%s] attribute of the existing index is [%s] which conflicts with the new [%s], resetting the attribute to None c@s eZdZdS)DuplicateWarningNrbrFrFrFrGrisriz; duplicate entries in table, taking most recently appended z your performance may suffer as PyTables will pickle object types that it cannot map directly to c-types [inferred_type->%s,key->%s] [items->%s] fixedtable)frjtrkz; : boolean drop ALL nan rows when appending to a table z~ : format default format writing format, if None, then put will default to 'fixed' and append will default to 'table' zio.hdfZ dropna_tableF)Z validatordefault_format)rjrkNcCsLtdurHddl}|att|jjdkaWdn1s>0YtS)Nrstrict) _table_modtablesrAttributeErrorfileZ_FILE_OPEN_POLICY!_table_file_open_policy_is_strict)rqrFrFrG_tabless  ruaTrorNDataFrame | Series int | None str | Noneboolint | dict[str, int] | None bool | None Literal[True] | list[str] | NoneNone)keyvaluemode complevelcomplibappendformatindex min_itemsizedropna data_columnserrorsrLreturnc s|r$ f dd}n f dd}t|}t|trt||||d}||Wdq1s0Yn||dS)z+store this object, close it if we opened itc s|j d S)N)rrrnan_reprrrrLrstore rrrLrrrrrrrrFrGszto_hdf..c s|j d S)N)rrrrrrrLrputrrrFrGr(s)rrrN)r9rArNHDFStore) path_or_bufrrrrrrrrrrrrrrLrlrrFrrGto_hdfs   (rr)rrstartstop chunksizec  Ks|dvrtd|d|dur,t|dd}t|trN|jsDtd|} d} ntt|}t|tshtd zt j |} Wnt tfyd} Yn0| st d |d t|f||d | } d } zt|dur| }t|dkrtd|d}|ddD]}t||stdq|j}| j||||||| | dWStt tfyt|tstt| Wdn1s0YYn0dS)a) Read from the store, close it if we opened it. Retrieve pandas object stored in file, optionally based on where criteria. .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- path_or_buf : str, path object, pandas.HDFStore Any valid string path is acceptable. Only supports the local file system, remote URLs and file-like objects are not supported. If you want to pass in a path object, pandas accepts any ``os.PathLike``. Alternatively, pandas accepts an open :class:`pandas.HDFStore` object. key : object, optional The group identifier in the store. Can be omitted if the HDF file contains a single pandas object. mode : {'r', 'r+', 'a'}, default 'r' Mode to use when opening the file. Ignored if path_or_buf is a :class:`pandas.HDFStore`. Default is 'r'. errors : str, default 'strict' Specifies how encoding and decoding errors are to be handled. See the errors argument for :func:`open` for a full list of options. where : list, optional A list of Term (or convertible) objects. start : int, optional Row number to start selection. stop : int, optional Row number to stop selection. columns : list, optional A list of columns names to return. iterator : bool, optional Return an iterator object. chunksize : int, optional Number of rows to include in an iteration when using an iterator. **kwargs Additional keyword arguments passed to HDFStore. Returns ------- item : object The selected object. Return type depends on the object stored. See Also -------- DataFrame.to_hdf : Write a HDF file from a DataFrame. HDFStore : Low-level access to HDF files. Examples -------- >>> df = pd.DataFrame([[1, 1.0, 'a']], columns=['x', 'y', 'z']) # doctest: +SKIP >>> df.to_hdf('./store.h5', 'data') # doctest: +SKIP >>> reread = pd.read_hdf('./store.h5') # doctest: +SKIP )rr+rvzmode zG is not allowed while performing a read. Allowed modes are r, r+ and a.NrUrSz&The HDFStore must be open for reading.Fz5Support for generic buffers has not been implemented.zFile z does not exist)rrTrz]Dataset(s) incompatible with Pandas data types, not table, or no datasets found in HDF5 file.z?key must be provided when HDF5 file contains multiple datasets.)r_rrcolumnsiteratorr auto_close) ValueErrorr`rAris_openOSErrorr9rNNotImplementedErrorospathexists TypeErrorFileNotFoundErrorgroupsr^_is_metadata_of _v_pathnameselectKeyErrorrrrclose)rrrrr_rrrrrkwargsrrrrZcandidate_only_groupZgroup_to_checkrFrFrGread_hdf?sjO           (rr>)group parent_grouprcCsF|j|jkrdS|}|jdkrB|j}||kr:|jdkr:dS|j}qdS)zDCheck if a given group is a metadata group for a given parent_group.FrUmetaT)Z_v_depthZ _v_parent_v_name)rrcurrentparentrFrFrGrs  rc@seZdZUdZded<ded<ded<ded <ddd ddddZddZeddZeddZ ddddZ ddddZ ddddZ dddd Z ddd!d"d#Zdd$d%d&Zdd$d'd(Zd)d*Zd+d,Zddd.d/d0d1Zd2d3Zd4d5ZeZddd6d7d8Zd9d:Zedd$d;d<Zddd=d>d?Zddd@dAZddddBdCdDZddd d dEdFdGZdddd d dHdIdJZdddKdLdMZdddPd dQdRddddSdTdUZ ddddVdWZ!dddPd dQdXdRddYdZd[Z"dd\d]d^d_Z#ddd d`dadbdcZ$dddeZ%ddgdhZ&ddid!djdkZ'ddld!dmdnZ(ddd ddpdqdrZ)dd$dsdtZ*dudvZ+dddwdxdyZ,dd{dddld|d}d~Z-dddPd dQddddddZ.ddddZ/ddddddZ0ddd!ddZ1d S)raa Dict-like IO interface for storing pandas objects in PyTables. Either Fixed or Table format. .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- path : str File path to HDF5 file. mode : {'a', 'w', 'r', 'r+'}, default 'a' ``'r'`` Read-only; no data can be modified. ``'w'`` Write; a new file is created (an existing file with the same name would be deleted). ``'a'`` Append; an existing file is opened for reading and writing, and if the file does not exist it is created. ``'r+'`` It is similar to ``'a'``, but the file must already exist. complevel : int, 0-9, default None Specifies a compression level for data. A value of 0 or None disables compression. complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib' Specifies the compression library to be used. As of v0.20.2 these additional compressors for Blosc are supported (default if no compressor specified: 'blosc:blosclz'): {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy', 'blosc:zlib', 'blosc:zstd'}. Specifying a compression library which is not available issues a ValueError. fletcher32 : bool, default False If applying compression use the fletcher32 checksum. **kwargs These parameters will be passed to the PyTables open_file method. Examples -------- >>> bar = pd.DataFrame(np.random.randn(10, 4)) >>> store = pd.HDFStore('test.h5') >>> store['foo'] = bar # write to HDF5 >>> bar = store['foo'] # retrieve >>> store.close() **Create or load HDF5 file in-memory** When passing the `driver` option to the PyTables open_file method through **kwargs, the HDF5 file is loaded or created in-memory and will only be written when closed: >>> bar = pd.DataFrame(np.random.randn(10, 4)) >>> store = pd.HDFStore('test.h5', driver='H5FD_CORE') >>> store['foo'] = bar >>> store.close() # only now, data is written to disk z File | None_handlerN_moderR _complevelrz _fletcher32rvNFrx)rr fletcher32cKsd|vrtdtd}|dur@||jjvr@td|jjd|durX|durX|jj}t||_|durnd}||_d|_|r|nd|_ ||_ ||_ d|_ |j fd|i|dS) Nrz-format is not a defined argument for HDFStorerqzcomplib only supports z compression.rvrr)rrfiltersZ all_complibsZdefault_complibr9_pathrrr_complibr_filtersopen)selfrrrrrrrqrFrFrG__init__/s&  zHDFStore.__init__cCs|jSrIrrrFrFrG __fspath__QszHDFStore.__fspath__cCs||jdusJ|jjS)zreturn the root nodeN)_check_if_openrrootrrFrFrGrTsz HDFStore.rootcCs|jSrIrrrFrFrGfilename[szHDFStore.filenamercCs ||SrI)getrrrFrFrG __getitem___szHDFStore.__getitem__cCs|||dSrIr)rrrrFrFrG __setitem__bszHDFStore.__setitem__cCs ||SrI)removerrFrFrG __delitem__eszHDFStore.__delitem__rOc CsDz ||WSttfy"Yn0tdt|jd|ddS)z$allow attribute access to get stores'z' object has no attribute 'N)rrrfrrtyperc)rrPrFrFrG __getattr__hs zHDFStore.__getattr__rrcCs8||}|dur4|j}||ks0|dd|kr4dSdS)zx check for existence of this key can match the exact pathname or the pathnm w/o the leading '/' NrUTF)get_noder)rrnoderPrFrFrG __contains__rs  zHDFStore.__contains__rcCs t|SrI)r^rrrFrFrG__len__~szHDFStore.__len__cCst|j}t|d|dS)N File path:  )r;rr)rpstrrFrFrG__repr__s zHDFStore.__repr__cCs|SrIrFrrFrFrG __enter__szHDFStore.__enter__cCs |dSrI)r)rexc_type exc_value tracebackrFrFrG__exit__szHDFStore.__exit__pandas list[str])includercCs^|dkrdd|DS|dkrJ|jdus0Jdd|jjddd DStd |d dS) a# Return a list of keys corresponding to objects stored in HDFStore. Parameters ---------- include : str, default 'pandas' When kind equals 'pandas' return pandas objects. When kind equals 'native' return native HDF5 Table objects. .. versionadded:: 1.1.0 Returns ------- list List of ABSOLUTE path-names (e.g. have the leading '/'). Raises ------ raises ValueError if kind has an illegal value rcSsg|] }|jqSrFrrWnrFrFrGr[z!HDFStore.keys..nativeNcSsg|] }|jqSrFrrrFrFrGr[s/Table) classnamez8`include` should be either 'pandas' or 'native' but is 'r)rrZ walk_nodesr)rrrFrFrGkeyss z HDFStore.keyscCs t|SrI)iterrrrFrFrG__iter__szHDFStore.__iter__ccs|D]}|j|fVqdS)z' iterate on key->group N)rr)rgrFrFrGitemss zHDFStore.items)rcKst}|j|krR|jdvr$|dvr$n(|dvrL|jrLtd|jd|jd||_|jr`||jr|jdkrtj|j|j|j d|_ t r|jrd }t ||j |j|jfi||_d S) a9 Open the file in the specified mode Parameters ---------- mode : {'a', 'w', 'r', 'r+'}, default 'a' See HDFStore docstring or tables.open_file for info about modes **kwargs These parameters will be passed to the PyTables open_file method. )rvw)rr)rzRe-opening the file [z ] with mode [z] will delete the current file!r)rzGCannot open HDF5 file, which is already opened, even in read-only mode.N)rurrrarrrFiltersrrrrtr open_filer)rrrrqmsgrFrFrGrs*    z HDFStore.opencCs|jdur|jd|_dS)z0 Close the PyTables file handle N)rrrrFrFrGrs  zHDFStore.closecCs|jdurdSt|jjS)zF return a boolean indicating whether the file is open NF)rrzZisopenrrFrFrGrs zHDFStore.is_open)fsynccCsT|jdurP|j|rPtt t|jWdn1sF0YdS)a Force all buffered modifications to be written to disk. Parameters ---------- fsync : bool (default False) call ``os.fsync()`` on the file handle to force writing to disk. Notes ----- Without ``fsync=True``, flushing may not guarantee that the OS writes to disk. With fsync, the operation will block until the OS claims the file has been written; however, other caching layers may still interfere. N)rflushrrrrfileno)rrrFrFrGrs    zHDFStore.flushcCsVt<||}|dur*td|d||WdS1sH0YdS)z Retrieve pandas object stored in file. Parameters ---------- key : str Returns ------- object Same type as object stored in file. NNo object named in the file)rrr _read_grouprrrrFrFrGr s  z HDFStore.get)rrc st||} | dur"td|dt|dd}|| fdd} t|| |j|||||d } | S) a Retrieve pandas object stored in file, optionally based on where criteria. .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- key : str Object being retrieved from file. where : list or None List of Term (or convertible) objects, optional. start : int or None Row number to start selection. stop : int, default None Row number to stop selection. columns : list or None A list of columns that if not None, will limit the return columns. iterator : bool or False Returns an iterator. chunksize : int or None Number or rows to include in iteration, return an iterator. auto_close : bool or False Should automatically close the store when finished. Returns ------- object Retrieved object from file. NrrrUrScsj|||dS)N)rrr_rread_start_stop_whererrErFrGfuncZszHDFStore.select..funcr_nrowsrrrrr)rrr`_create_storer infer_axes TableIteratorr get_result) rrr_rrrrrrrritrFrrGr"s(.    zHDFStore.selectrrrcCs8t|dd}||}t|ts(td|j|||dS)a return the selection as an Index .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- key : str where : list of Term (or convertible) objects, optional start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection rUrSz&can only read_coordinates with a tabler_rr)r` get_storerrArrread_coordinates)rrr_rrtblrFrFrGselect_as_coordinatesms    zHDFStore.select_as_coordinates)rcolumnrrcCs,||}t|tstd|j|||dS)a~ return a single column from the table. This is generally only useful to select an indexable .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- key : str column : str The column of interest. start : int or None, default None stop : int or None, default None Raises ------ raises KeyError if the column is not found (or key is not a valid store) raises ValueError if the column can not be extracted individually (it is part of a data block) z!can only read_column with a tablerrr)r rArr read_column)rrrrrrrFrFrG select_columns#  zHDFStore.select_column)rc  szt|dd}t|ttfr.t|dkr.|d}t|trRj||||||| dSt|ttfshtdt|sxtd|dur|d}fdd |D |} d} t | |fgt |D]\\} } | durt d | d | jstd | jd | dur | j} q| j| krtdqdd D}tdd|Ddfdd}t| ||| ||||| d }|jddS)a Retrieve pandas objects from multiple tables. .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- keys : a list of the tables selector : the table to apply the where criteria (defaults to keys[0] if not supplied) columns : the columns I want back start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection iterator : bool, return an iterator, default False chunksize : nrows to include in iteration, return an iterator auto_close : bool, default False Should automatically close the store when finished. Raises ------ raises KeyError if keys or selector is not found or keys is empty raises TypeError if keys is not a list or tuple raises ValueError if the tables are not ALL THE SAME DIMENSIONS rUrSr)rr_rrrrrrzkeys must be a list/tuplez keys must have a non-zero lengthNcsg|]}|qSrF)r rWkrrFrGr[rz/HDFStore.select_as_multiple..zInvalid table []zobject [z>] is not a table, and cannot be used in all select as multiplez,all tables must have exactly the same nrows!cSsg|]}t|tr|qSrF)rArrWxrFrFrGr[rcSsh|]}|jddqSr)non_index_axesrWrmrFrFrG rz.HDFStore.select_as_multiple..cs*fddD}t|ddS)Ncsg|]}|jdqS)r_rrrrr)rrrrrFrGr[sz=HDFStore.select_as_multiple..func..F)axisverify_integrity)r- _consolidate)rrrobjs)rrtblsrrGrsz)HDFStore.select_as_multiple..funcrT coordinates)r`rAr\r]r^rNrrrr  itertoolschainzipris_tablepathnamerrr )rrr_selectorrrrrrrrErrmrZ_tblsrr rF)rrrr#rGselect_as_multiplesd+           zHDFStore.select_as_multipleTrorwr{r})rrrrrr track_timesrcCsH|durtdpd}||}|j||||||||| | | | | |ddS)aO Store object in HDFStore. Parameters ---------- key : str value : {Series, DataFrame} format : 'fixed(f)|table(t)', default is 'fixed' Format to use when storing object in HDFStore. Value can be one of: ``'fixed'`` Fixed format. Fast writing/reading. Not-appendable, nor searchable. ``'table'`` Table format. Write as a PyTables Table structure which may perform worse but allow more flexible operations like searching / selecting subsets of the data. append : bool, default False This will force Table format, append the input data to the existing. data_columns : list of columns or True, default None List of columns to create as data columns, or True to use all columns. See `here `__. encoding : str, default None Provide an encoding for strings. track_times : bool, default True Parameter is propagated to 'create_table' method of 'PyTables'. If set to False it enables to have the same h5 files (same hashes) independent on creation time. .. versionadded:: 1.1.0 Nio.hdf.default_formatrj) rrrrrrrrrLrr-r)r_validate_format_write_to_group)rrrrrrrrrrrrLrr-rrFrFrGr/s&0  z HDFStore.putc Cst|dd}z||}Wnty.Yntty@Ynbty}zJ|dur`td|||}|dur|jddWYd}~dSWYd}~n d}~00t |||r|j jddn|j std|j |||dSdS) a: Remove pandas object partially by specifying the where condition Parameters ---------- key : str Node to remove or delete rows from where : list of Term (or convertible) objects, optional start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection Returns ------- number of rows removed (or None if not a Table) Raises ------ raises KeyError if key is not a valid store rUrSNz5trying to remove a node with a non-None where clause!T recursivez7can only remove with where on objects written as tablesr ) r`r rAssertionError ExceptionrrZ _f_removecomall_nonerr)delete)rrr_rrrEerrrrFrFrGrss2     $zHDFStore.remover|)rrrrrrrcCsl| durtd|dur td}|dur4tdp2d}||}|j||||||||| | | | ||||ddS)a6 Append to Table in file. Node must already exist and be Table format. Parameters ---------- key : str value : {Series, DataFrame} format : 'table' is the default Format to use when storing object in HDFStore. Value can be one of: ``'table'`` Table format. Write as a PyTables Table structure which may perform worse but allow more flexible operations like searching / selecting subsets of the data. append : bool, default True Append the input data to the existing. data_columns : list of columns, or True, default None List of columns to create as indexed data columns for on-disk queries, or True to use all columns. By default only the axes of the object are indexed. See `here `__. min_itemsize : dict of columns that specify minimum str sizes nan_rep : str to use as str nan representation chunksize : size to chunk the writing expectedrows : expected TOTAL row size of this table encoding : default None, provide an encoding for str dropna : bool, default False Do not write an ALL nan row to the store settable by the option 'io.hdf.dropna_table'. Notes ----- Does *not* check if data being appended overlaps with existing data in the table, so be careful Nz>columns is not a supported keyword in append, try data_columnszio.hdf.dropna_tabler.rk)raxesrrrrrrr expectedrowsrrrLr)rrr/r0)rrrrr9rrrrrrrrr:rrrLrrFrFrGrs68  zHDFStore.appenddict)dc s|durtdt|ts"td||vr2tdtttjttt d}d} g} | D]0\} dur| durtd| } qh| qh| durֈj |} | t| } t| | } | | || <|dur||}|r*fdd|D}t|}|D]}||}qj||d d}| D]h\} | |krT|nd}j|d }|durfd d | Dnd}|j| |f||d |q>dS)a Append to multiple tables Parameters ---------- d : a dict of table_name to table_columns, None is acceptable as the values of one node (this will get all the remaining columns) value : a pandas object selector : a string that designates the indexable table; all of its columns will be designed as data_columns, unless data_columns is passed, in which case these are used data_columns : list of columns to create as data columns, or True to use all columns dropna : if evaluates to True, drop rows from all tables if any single row in each table has all NaN. Default False. Notes ----- axes parameter is currently not accepted Nztaxes is currently not accepted as a parameter to append_to_multiple; you can create the tables independently insteadzQappend_to_multiple must have a dictionary specified as the way to split the valuez=append_to_multiple requires a selector that is in passed dictrzLrz.HDFStore.append_to_multiple..rrcsi|]\}}|vr||qSrFrFrWrr)vrFrG \rz/HDFStore.append_to_multiple..)rr)rrAr;rr\setrangendim _AXES_MAPrrextendr9 differencer(sorted get_indexertakevaluesnext intersectionlocpopreindexr)rr<rr+rr9rrrZ remain_keyZ remain_valuesrorderedZorddZidxsZ valid_indexrrdcvalfilteredrF)rCrrGappend_to_multiplesZ &    zHDFStore.append_to_multiplery)roptlevelkindcCsBt||}|durdSt|ts.td|j|||ddS)a Create a pytables index on the table. Parameters ---------- key : str columns : None, bool, or listlike[str] Indicate which columns to create an index on. * False : Do not create any indexes. * True : Create indexes on all columns. * None : Create indexes on all columns. * listlike : Create indexes on the given columns. optlevel : int or None, default None Optimization level, if None, pytables defaults to 6. kind : str or None, default None Kind of index, if None, pytables defaults to "medium". Raises ------ TypeError: raises if the node is not a table Nz1cannot create table index on a Fixed format store)rrYrZ)rur rArr create_index)rrrrYrZrErFrFrGcreate_table_indexbs  zHDFStore.create_table_indexcCs<t||jdusJtdus(Jdd|jDS)z Return a list of all the top-level nodes. Each node returned is not a pandas storage object. Returns ------- list List of objects. NcSsPg|]H}t|tjjst|jddsHt|ddsHt|tjjr|jdkr|qS) pandas_typeNrk) rArplinkLinkgetattr_v_attrsrkrr)rWrrFrFrGr[s  z#HDFStore.groups..)rurrrp walk_groupsrrFrFrGrs  zHDFStore.groupsrccst||jdusJtdus(J|j|D]}t|jdddurLq4g}g}|jD]B}t|jdd}|durt |tj j r| |j q^| |j q^|jd||fVq4dS)aS Walk the pytables group hierarchy for pandas objects. This generator will yield the group path, subgroups and pandas object names for each group. Any non-pandas PyTables objects that are not a group will be ignored. The `where` group itself is listed first (preorder), then each of its child groups (following an alphanumerical order) is also traversed, following the same procedure. Parameters ---------- where : str, default "/" Group where to start walking. Yields ------ path : str Full path to a group (without trailing '/'). groups : list Names (strings) of the groups contained in `path`. leaves : list Names (strings) of the pandas objects contained in `path`. Nr]r)rurrrprbr`raZ _v_childrenrNrArGrouprrrrstrip)rr_rrleaveschildr]rFrFrGwalks  z HDFStore.walkz Node | NonecCs~||dsd|}|jdus(Jtdus4Jz|j|j|}Wntjjy`YdS0t|tj szJt ||S)z9return the node with the key or None if it does not existrN) r startswithrrprr exceptionsZNoSuchNodeErrorrAr>r)rrrrFrFrGrs  zHDFStore.get_nodeGenericFixed | TablecCs8||}|dur"td|d||}||S)z| | vrj|rj| | || } t| trd} |rdd| j D} | j | | | t | dd| j dq>| j | | | j dq>| S) a; Copy the existing store to a new file, updating in place. Parameters ---------- propindexes : bool, default True Restore indexes in copied file. keys : list, optional List of keys to include in the copy (defaults to all). overwrite : bool, default True Whether to overwrite (remove and replace) existing nodes in the new store. mode, complib, complevel, fletcher32 same as in HDFStore.__init__ Returns ------- open file handle of the new store )rrrrNFcSsg|]}|jr|jqSrF) is_indexedrPrWrvrFrFrGr[ rz!HDFStore.copy..r)rrrLrK)rr\rrAr]r rrrr9rr`rLr)rrsrrkrrrr overwriteZ new_storerrEdatarrFrFrGcopys6       z HDFStore.copyc Cs t|j}t|d|d}|jrt|}t|rg}g}|D]}z<||}|dur|t|j pj||t|p|dWqDt yYqDt y}z0||t|} |d| dWYd}~qDd}~00qD|t d||7}n|d7}n|d 7}|S) zg Print detailed information on the store. Returns ------- str rrNzinvalid_HDFStore nodez[invalid_HDFStore node: r EmptyzFile is CLOSED) r;rrrrKrr^r rr*r3r4r:) rroutputZlkeysrrNrrEZdetailZdstrrFrFrGinfo-s.     * z HDFStore.infocCs|jst|jddS)Nz file is not open!)rrfrrrFrFrGrWszHDFStore._check_if_open)rrc CsLzt|}Wn6tyF}ztd|d|WYd}~n d}~00|S)zvalidate / deprecate formatsz#invalid HDFStore format specified [rN) _FORMAT_MAPlowerrr)rrr8rFrFrGr/[s (zHDFStore._validate_formatr@zDataFrame | Series | None)rrLrrc s"durtttfstdfdd}ttjdd}ttjdd}|durƈdurttdustJtddsttj j rd}d }qtd n$ttrd }nd }dkr|d 7}d|vr(t t d} z | |} Wn0t y} z|d| WYd} ~ n d} ~ 00| |||dS|durˆdur|dkrtdd} | dur| jdkrnd}n| jdkrd}nB|dkrtdd} | dur| jdkrd}n| jdkrd}ttttttd} z | |} Wn0t y} z|d| WYd} ~ n d} ~ 00| |||dS)z"return a suitable class to operateNz(value must be None, Series, or DataFramec s$td|ddtdS)Nz(cannot properly create the storer for: [z ] [group->,value->z ,format->)rr)rmrrrrFrGerrorssz&HDFStore._create_storer..errorr] table_typerk frame_table generic_tablezKcannot create a storer if the object is not existing nor a value are passedseriesframe_table)r}r~ _STORER_MAPrLr series_tablerrUappendable_seriesappendable_multiseriesappendable_frameappendable_multiframe)r|rrrrworm _TABLE_MAP)rAr+r&rrHr`rarurprkr SeriesFixed FrameFixedrnlevels GenericTableAppendableSeriesTableAppendableMultiSeriesTableAppendableFrameTableAppendableMultiFrameTable WORMTable)rrrrrLrryptttrclsr8rrrFrxrGresr                     zHDFStore._create_storerr~)rrrrrr-rcCst|ddr|dks|rdS|||}|j|||||d}|rr|jrZ|jrb|dkrb|jrbtd|jsz|n||js|rtd|j|||||| | | | | |||d t|t r|r|j |ddS) NemptyrkrrjzCan only append to Tablesz0Compression not supported on Fixed format stores) objr9rrrrrrr:rrrr-)r) r`_identify_grouprr) is_existsrset_object_infowriterArr[)rrrrr9rrrrrrrr:rrrrLrr-rrErFrFrGr0s:   zHDFStore._write_to_groupr>rcCs||}||SrI)rrr)rrrErFrFrGrs zHDFStore._read_group)rrrcCsN||}|jdusJ|dur8|s8|jj|ddd}|durJ||}|S)z@Identify HDF5 group based on key, delete/create group if needed.NTr1)rr remove_node_create_nodes_and_group)rrrrrFrFrGrs   zHDFStore._identify_groupcCsv|jdusJ|d}d}|D]P}t|s.q |}|dsD|d7}||7}||}|durl|j||}|}q |S)z,Create nodes from key and return group name.Nr)rsplitr^endswithrZ create_group)rrpathsrpnew_pathrrFrFrGrs   z HDFStore._create_nodes_and_group)rvNNF)r)rv)F)NNNNFNF)NNN)NN)NNNNNFNF) NTFNNNNNNroTF)NNN)NNTTNNNNNNNNNNro)NNF)NNN)r)rTNNNFT)NNr@ro)NTFNNNNNNFNNNroT)2rcrdre__doc____annotations__rrpropertyrrrrrrrrrrrrrr iteritemsrrrrrrrrr,rrrrXr\rrgrr rprtrr/rr0rrrrFrFrFrGrs A"    "-N$+~D=Zd( 0 =* a>rc@sbeZdZUdZded<ded<ded<dddd dd d d d ZddZddZdd dddZdS)raa Define the iteration interface on a table Parameters ---------- store : HDFStore s : the referred storer func : the function to execute the query where : the where of the query nrows : the rows to iterate on start : the passed start value (default is None) stop : the passed stop value (default is None) iterator : bool, default False Whether to use the default iterator. chunksize : the passed chunking value (default is 100000) auto_close : bool, default False Whether to automatically close the store at the end of iteration. rxrrrrjrENFrz)rrErrrc Cs||_||_||_||_|jjrN|dur,d}|dur8d}|durD|}t||}||_||_||_d|_ |sr| dur| dur~d} t | |_ nd|_ | |_ dS)Nr順) rrErr_r)minrrrr%rRrr) rrrErr_rrrrrrrFrFrGrDs,    zTableIterator.__init__ccsv|j}|jdurtd||jkrjt||j|j}|dd|j||}|}|dust|sbq|Vq|dS)Nz*Cannot iterate until get_result is called.) rr%rrrrrr^r)rrrrrFrFrGrns  zTableIterator.__iter__cCs|jr|jdSrI)rrrrrFrFrGr~szTableIterator.closer$cCs|jdur4t|jtstd|jj|jd|_|S|rft|jtsLtd|jj|j|j|j d}n|j}| |j|j |}| |S)Nz0can only use an iterator or chunksize on a table)r_z$can only read_coordinates on a tabler ) rrArErrrr_r%rrrr)rr%r_resultsrFrFrGr s    zTableIterator.get_result)NNFNF)F) rcrdrerrrrrr rFrFrFrGr,s  *rc @steZdZUdZdZdZgdZded<ded<dHddd d d Ze d d ddZ e dd ddZ d dddZ dd ddZ dddddZdd ddZe dd ddZd ddd!d"d#Zd$d%Ze d&d'Ze d(d)Ze d*d+Ze d,d-Zd.d/ZdId0d1Zd2d3Zd4dd5d6d7ZdJd8d9Zdd:d;d<Zd=d>Zd?d@ZdAdBZd4dCdDdEZ d4dCdFdGZ!dS)KIndexCola an index column description class Parameters ---------- axis : axis which I reference values : the ndarray like converted values kind : a string description of this type typ : the pytables type pos : the position in the pytables T)freqtz index_namerNrPcnameNry)rPrcCst|tstd||_||_||_||_|p0||_||_||_ ||_ | |_ | |_ | |_ | |_| |_||_|dur|||t|jtsJt|jtsJdS)Nz`name` must be a str.)rArNrrNrZtyprPrrposrrrrTrkrmetadataset_pos)rrPrNrZrrrrrrrrTrkrrrFrFrGrs(   zIndexCol.__init__rRrcCs|jjSrI)ritemsizerrFrFrGrszIndexCol.itemsizecCs |jdS)N_kindrOrrFrFrG kind_attrszIndexCol.kind_attr)rcCs$||_|dur |jdur ||j_dS)z,set the position of this column in the TableN)rrZ_v_pos)rrrFrFrGrszIndexCol.set_poscCs@ttt|j|j|j|j|jf}dddt gd|DS)N,cSsg|]\}}|d|qSz->rFrBrFrFrGr[sz%IndexCol.__repr__..)rPrrrrZ) r]mapr;rPrrrrZjoinr(rtemprFrFrGrs zIndexCol.__repr__rrzotherrcstfdddDS)compare 2 col itemsc3s&|]}t|dt|dkVqdSrIr`rmrrrFrGr@sz"IndexCol.__eq__..)rPrrrr=rrrFrrG__eq__szIndexCol.__eq__cCs || SrI)rrrFrFrG__ne__szIndexCol.__ne__cCs"t|jdsdSt|jj|jjS)z%return whether I am an indexed columnr?F)hasattrrkr`r?rrlrrFrFrGrls zIndexCol.is_indexed np.ndarrayrNrLrc Cst|tjsJt||jjdur.||j}t|j}t ||||}i}t|j |d<|j durpt|j |d<t }t |jst|jrt}n|jdkrd|vrdd}z||fi|}Wn2tyd|vrd|d<||fi|}Yn0t||j} | | fS)zV Convert the data from this selection to the appropriate pandas type. NrPri8c[stfd|i|S)NZordinal)r*)rkwdsrFrFrGr$sz"IndexCol.convert..)rArBndarrayrdtypefieldsrrHrZ_maybe_convertrrr(rrr'r_set_tzr) rrNrrLrval_kindrfactoryZ new_pd_indexZfinal_pd_indexrFrFrGconvert s,      zIndexCol.convertcCs|jS)zreturn the valuesrNrrFrFrG take_data4szIndexCol.take_datacCs|jjSrI)rkrarrFrFrGattrs8szIndexCol.attrscCs|jjSrIrk descriptionrrFrFrGr<szIndexCol.descriptioncCst|j|jdS)z!return my current col descriptionN)r`rrrrFrFrGcol@sz IndexCol.colcCs|jSzreturn my cython valuesrrrFrFrGcvaluesEszIndexCol.cvaluescCs t|jSrI)rrNrrFrFrGrJszIndexCol.__iter__cCsPt|jdkrLt|tr$||j}|durL|jj|krLtj ||j d|_dS)z maybe set a string col itemsize: min_itemsize can be an integer or a dict with this columns name with an integer size stringN)rr) rHrZrAr;rrPrrru StringColr)rrrFrFrGmaybe_set_sizeMs   zIndexCol.maybe_set_sizecCsdSrIrFrrFrFrGvalidate_namesZszIndexCol.validate_namesAppendableTable)handlerrcCs:|j|_||||||||dSrI)rk validate_col validate_attrvalidate_metadatawrite_metadataset_attr)rrrrFrFrGvalidate_and_set]s    zIndexCol.validate_and_setcCs^t|jdkrZ|j}|durZ|dur*|j}|j|krTtd|d|jd|jd|jSdS)z:validate this column: return the compared against itemsizerNz#Trying to store a string with len [z] in [z)] column but this column has a limit of [zC]! Consider using min_itemsize to preset the sizes on these columns)rHrZrrrr)rrcrFrFrGres zIndexCol.validate_colrcCsB|r>t|j|jd}|dur>||jkr>td|d|jddS)Nzincompatible kind in col [ - r)r`rrrZr)rrZ existing_kindrFrFrGrxs zIndexCol.validate_attrc Cs|jD]}t||d}||ji}||}||vr|dur||kr|dvrt|||f}tj|tt dd||<t ||dqt d|jd|d|d|d q|dus|dur|||<qdS) z set/update the info for this indexable with the key/value if there is a conflict raise/warn as needed N)rr stacklevelzinvalid info for [z] for [z], existing_value [z] conflicts with new value [r) _info_fieldsr` setdefaultrPrattribute_conflict_docwarningswarnrhrsetattrr)rrtrridxZexisting_valuewsrFrFrG update_infos*   zIndexCol.update_infocCs$||j}|dur |j|dS)z!set my state from the passed infoN)rrP__dict__update)rrtrrFrFrGset_infos zIndexCol.set_infocCst|j|j|jdS)zset the kind for this columnN)rrrrZrrFrFrGrszIndexCol.set_attr)rcCsB|jdkr>|j}||j}|dur>|dur>t||s>tddS)z:validate that kind=category does not change the categoriescategoryNzEcannot append a categorical with different categories to the existing)rr read_metadatarr%r)rrZ new_metadataZ cur_metadatarFrFrGrs  zIndexCol.validate_metadatacCs|jdur||j|jdS)zset the meta dataN)rrr)rrrFrFrGrs zIndexCol.write_metadata) NNNNNNNNNNNNN)N)N)"rcrdreris_an_indexableis_data_indexablerrrrrrrrrrrlrrrrrrrrrrrrrrrrrrFrFrFrGrsf  , +       !rc@s<eZdZdZeddddZddddd d Zd d Zd S)GenericIndexColz:an index which is not represented in the data of the tablerzrcCsdSNFrFrrFrFrGrlszGenericIndexCol.is_indexedrrNrcCs2t|tjsJt|ttt|}||fS)z Convert the data from this selection to the appropriate pandas type. Parameters ---------- values : np.ndarray nan_rep : str encoding : str errors : str )rArBrrr/aranger^)rrNrrLrrFrFrGrs zGenericIndexCol.convertcCsdSrIrFrrFrFrGrszGenericIndexCol.set_attrN)rcrdrerrrlrrrFrFrFrGrs rc s,eZdZdZdZdZddgZd9dddfd d Zedd d d Z edd ddZ dd ddZ dddddZ ddddZ ddZeddddd Zed!d"Zedd#d$d%d&Zeddd$d'd(Zed)d*Zed+d,Zed-d.Zed/d0Zd1d2Zd3ddd4d5d6Zd7d8ZZS):DataCola3 a data holding column, by definition this is not indexable Parameters ---------- data : the actual data cname : the column name in the table to hold the data (typically values) meta : a string description of the metadata metadata : the actual metadata FrrTNrNzDtypeArg | None)rPrc s2tj||||||||| | | d | |_| |_dS)N) rPrNrZrrrrrTrkrr)superrrro)rrPrNrZrrrrrTrkrrrro __class__rFrGrs zDataCol.__init__rcCs |jdS)N_dtyperOrrFrFrG dtype_attr szDataCol.dtype_attrcCs |jdS)N_metarOrrFrFrG meta_attr szDataCol.meta_attrcCs@ttt|j|j|j|j|jf}dddt gd|DS)NrcSsg|]\}}|d|qSrrFrBrFrFrGr[ sz$DataCol.__repr__..)rPrrrZshape) r]rr;rPrrrZrrr(rrFrFrGr s zDataCol.__repr__rrzrcstfdddDS)rc3s&|]}t|dt|dkVqdSrIrrmrrFrGr@& sz!DataCol.__eq__..)rPrrrrrrFrrGr$ szDataCol.__eq__rrocCs@|dus J|jdusJt|\}}||_||_t||_dSrI)r_get_data_and_dtype_namero_dtype_to_kindrZ)rro dtype_namerFrFrGset_data+ s   zDataCol.set_datacCs|jS)zreturn the datarrrFrFrGr5 szDataCol.take_datar<)rNrcCs|j}|j}|j}|jdkr&d|jf}t|trJ|j}|j||jj d}ntt |sZt |rf| |}nXt |rz||}nDt|rtj||dd}n&t|r|||}n|j||j d}|S)zW Get an appropriately typed and shaped pytables.Col object for values. rUrZrrr)rrrrGsizerAr0codes get_atom_datarPrrget_atom_datetime64r#get_atom_timedelta64rruZ ComplexColr"get_atom_string)rrNrrrratomrFrFrG _get_atom9 s$     zDataCol._get_atomcCstj||ddS)Nrr rurrrrrFrFrGrY szDataCol.get_atom_stringz type[Col]rZrcCsR|dr$|dd}d|d}n"|dr4d}n|}|d}tt|S)z0return the PyTables column class for this columnuintNZUIntr<periodInt64Col)rh capitalizer`ru)rrZZk4Zcol_nameZkcaprFrFrGget_atom_coltype] s    zDataCol.get_atom_coltypecCs|j|d|ddS)Nr rrrrrrZrFrFrGrl szDataCol.get_atom_datacCstj|ddSNrrrurrrrFrFrGrp szDataCol.get_atom_datetime64cCstj|ddSr!r"r#rFrFrGrt szDataCol.get_atom_timedelta64cCst|jddS)Nr)r`rorrFrFrGrx sz DataCol.shapecCs|jSrrrrFrFrGr| szDataCol.cvaluescCs`|r\t|j|jd}|dur2|t|jkr2tdt|j|jd}|dur\||jkr\tddS)zAvalidate that we have the same order as the existing & same dtypeNz4appended items do not match existing items in table!z@appended items dtype do not match existing items dtype in table!)r`rrr\rNrrr)rrZexisting_fieldsZexisting_dtyperFrFrGr szDataCol.validate_attrrrcCst|tjsJt||jjdur.||j}|jdus.cSsg|]}t|qSrFr fromtimestampr,rFrFrGr[ rr) categoriesrTFrpOrrrLr)"rArBrrrrrrrrrZrHrrrTrrasarrayobjectrravelr(Zfloat64r.anyastyperRZcumsum_valuesr0Z from_codesr_unconvert_string_arrayrN)rrNrrLr convertedr rZrrrTrrr0rmaskrFrFrGr sf              zDataCol.convertcCsHt|j|j|jt|j|j|j|jdus2Jt|j|j|jdS)zset the data for this columnN)rrrrNrrrrrrFrFrGr szDataCol.set_attr) NNNNNNNNNNNN)rcrdrerrrrrrrrrrr r classmethodrrrrrrrrrrr __classcell__rFrFrrGrsX          erc@sTeZdZdZdZddZeddZeddd d d Zed d Z eddZ dS)DataIndexableColz+represent a data column that can be indexedTcCst|jstddS)N-cannot have non-object label DataIndexableCol)r(rNZ is_objectrrrFrFrGr szDataIndexableCol.validate_namescCstj|dS)N)rrrrFrFrGr sz DataIndexableCol.get_atom_stringrNr<rcCs|j|dS)Nr rr rFrFrGr szDataIndexableCol.get_atom_datacCs tSrIr"r#rFrFrGr sz$DataIndexableCol.get_atom_datetime64cCs tSrIr"r#rFrFrGr sz%DataIndexableCol.get_atom_timedelta64N) rcrdrerrrr=rrrrrFrFrFrGr? s  r?c@seZdZdZdS)GenericDataIndexableColz(represent a generic pytables data columnN)rcrdrerrFrFrFrGrA srAc@seZdZUdZded<dZded<ded<ded <ded <d ed <d ed<ded<dZdLd d dddddZeddddZ eddddZ eddZ dddd Z d!d"Z d#d$Zed%d&Zed'd(Zed)d*Zed+d,Zeddd-d.Zeddd/d0Zed1d2Zd3d4Zd5d6Zed7d8Zeddd9d:Zed;d<Zd=d>ZdMd@dAZdBdCZdNdDdDdEdFdGZdHdIZdOdDdDdEdJdKZ d?S)PFixedz represent an object in my store facilitate read/write of various types of objects this is an abstract base class Parameters ---------- parent : HDFStore group : Node The group node where the table resides. rN pandas_kindrj format_typetype[DataFrame | Series]obj_typerRrGrLrrr>rrFr@ro)rrrLrcCsZt|tsJt|tdus"Jt|tjs:Jt|||_||_t||_||_ dSrI) rArrrpr>rrrMrLr)rrrrLrrFrFrGr3 s  zFixed.__init__rzrcCs*|jddko(|jddko(|jddkS)NrrU )versionrrFrFrGis_old_versionB szFixed.is_old_versionztuple[int, int, int]cCs`tt|jjdd}z0tdd|dD}t|dkrB|d}WntyZd}Yn0|S) zcompute and set our versionpandas_versionNcss|]}t|VqdSrIrRrrFrFrGr@K rz Fixed.version...rHr)rrr)rHr`rrar]rr^rr)rrIrFrFrGrIF s    z Fixed.versioncCstt|jjddS)Nr])rHr`rrarrFrFrGr]R szFixed.pandas_typecCs^||j}|durXt|ttfrDddd|D}d|d}|jdd|d S|jS) (return a pretty representation of myselfNrcSsg|] }t|qSrFr;rrFrFrGr[\ rz"Fixed.__repr__..[r12.12z (shape->))rrrAr\r]rr])rrEZjshaperFrFrGrV s zFixed.__repr__cCst|j|j_tt|j_dS)zset my pandas type & versionN)rNrCrr]_versionrKrrFrFrGra szFixed.set_object_infocCst|}|SrIr1)rZnew_selfrFrFrGrpf s z Fixed.copycCs|jSrI)rrrFrFrGrj sz Fixed.shapecCs|jjSrIrrrrFrFrGr*n szFixed.pathnamecCs|jjSrI)rrrrFrFrGrr sz Fixed._handlecCs|jjSrI)rrrrFrFrGrv szFixed._filterscCs|jjSrI)rrrrFrFrGrz szFixed._complevelcCs|jjSrI)rrrrFrFrGr~ szFixed._fletcher32cCs|jjSrI)rrarrFrFrGr sz Fixed.attrscCsdSzset our object attributesNrFrrFrFrG set_attrs szFixed.set_attrscCsdS)zget our object attributesNrFrrFrFrG get_attrs szFixed.get_attrscCs|jS)zreturn my storablerrrFrFrGstorable szFixed.storablecCsdSrrFrrFrFrGr szFixed.is_existscCst|jddS)Nr)r`rXrrFrFrGr sz Fixed.nrowscCs|dur dSdS)z%validate against an existing storableNTrFrrFrFrGvalidate szFixed.validateNcCsdS)+are we trying to operate on an old version?TrF)rr_rFrFrGvalidate_version szFixed.validate_versioncCs|j}|durdS|dS)zr infer the axes of my storer return a boolean indicating if we have a valid storer or not NFT)rXrW)rrErFrFrGr s zFixed.infer_axesrxrrcCs tddS)Nz>cannot read on an abstract storer: subclasses should implementrrr_rrrrFrFrGr sz Fixed.readcKs tddS)Nz?cannot write on an abstract storer: subclasses should implementr]rrrFrFrGr sz Fixed.writecCs0t|||r$|jj|jdddStddS)zs support fully deleting the node in its entirety (only) - where specification must be None Tr1Nz#cannot delete on an abstract storer)r5r6rrrr)rr_rrrFrFrGr7 sz Fixed.delete)r@ro)N)NNNN)NNN)!rcrdrerrrDr)rrrJrIr]rrrprr*rrrrrrVrWrXrrrYr[rrrr7rFrFrFrGrB sl                rBc@s(eZdZUdZedediZddeDZgZ de d<dd d d Z d d Z ddZ ddZedd ddZddZddZddZd:ddddddZd;dddd d!d"d#Zdd d$d%d&Zdd'd$d(d)Zddd4d5d6d7d8d9ZdS)? GenericFixedza generified fixed versiondatetimercCsi|]\}}||qSrFrF)rWrrCrFrFrGrD rzGenericFixed.r attributesrNrcCs|j|dS)N)_index_type_mapr)rrrFrFrG_class_to_alias szGenericFixed._class_to_aliascCst|tr|S|j|tSrI)rAr_reverse_index_maprr()raliasrFrFrG_alias_to_class s zGenericFixed._alias_to_classcCs|tt|dd}|tkr.d dd}|}n|tkrFd dd}|}n|}i}d|vrn|d|d<|turnt}d|vrt|dtr|d d|d<n |d|d<|tusJ||fS) N index_classrccSs:tj|j|d}tj|dd}|dur6|d|}|S)NrrOUTC)r1 _simple_newrNr' tz_localize tz_convert)rNrrZdtaresultrFrFrGrl s z*GenericFixed._get_index_factory..fcSstj||d}tj|ddS)NrjrO)r2rlr*)rNrrZparrrFrFrGrl srrzutf-8)NN)NN) rhrHr`r'r*r(r,rAbytesrD)rrrirlrrrFrFrG_get_index_factory s*     zGenericFixed._get_index_factorycCs$|durtd|dur tddS)zE raise if any keywords are passed which are not-None Nzqcannot pass a column specification when reading a Fixed format store. this store must be selected in its entiretyzucannot pass a where specification when reading from a Fixed format store. this store must be selected in its entirety)r)rrr_rFrFrG validate_read szGenericFixed.validate_readrzcCsdS)NTrFrrFrFrGr szGenericFixed.is_existscCs|j|j_|j|j_dSrU)rLrrrrFrFrGrV s zGenericFixed.set_attrsc CsRtt|jdd|_tt|jdd|_|jD]}t||tt|j|dq.dS)retrieve our attributesrLNrro)rMr`rrLrHrrbr)rrrFrFrGrW# s zGenericFixed.get_attrscKs |dSrI)rVrrrrFrFrGr* szGenericFixed.writeNrxr c Csddl}t|j|}|j}t|dd}t||jrD|d||}nztt|dd} t|dd} | durxtj| | d}n |||}| dkrt|d d} t || d d }n| d krtj |d d}|r|j S|SdS)z2read an array for the specified node (off of grouprN transposedF value_typerr)r$rTr%r'r() rqr`rrarAZVLArrayrHrBrrr4T) rrrrrqrrruretrrrrFrFrG read_array- s&      zGenericFixed.read_arrayr()rrrrcCshtt|j|d}|dkr.|j|||dS|dkrVt|j|}|j|||d}|Std|dS)N_varietymultir\regularzunrecognized index variety: )rHr`rread_multi_indexrread_index_noder)rrrrZvarietyrrrFrFrG read_indexO s zGenericFixed.read_index)rrcCst|tr,t|j|dd|||nt|j|ddtd||j|j}|||j t |j |}|j |j _ |j|j _t|ttfr|t||j _t|tttfr|j|j _t|tr|jdurt|j|j _dS)Nrzr{r|r)rAr)rrwrite_multi_index_convert_indexrLr write_arrayrNr`rrZrarPr'r*rerrir,rr_get_tz)rrrr;rrFrFrG write_index] s     zGenericFixed.write_indexr)c Cst|j|d|jtt|j|j|jD]\}\}}}t|rJt d|d|}t |||j |j }| ||jt|j|} |j| j_|| j_t| j|d|||d|} | | |q,dS)N_nlevelsz=Saving a MultiIndex with an extension dtype is not supported._level_name_label)rrr enumerater(levelsrnamesr rrrLrrrNr`rrZrarP) rrrilev level_codesrP level_keyZ conv_levelr label_keyrFrFrGrt s"  zGenericFixed.write_multi_indexcCst|j|d}g}g}g}t|D]l}|d|} t|j| } |j| ||d} || || j|d|} |j| ||d} || q&t|||ddS)Nrrr\rT)rrrr ) r`rrFrr~rrPryr))rrrrrrrrrrrrrrrFrFrGr} s      zGenericFixed.read_multi_indexr>)rrrrc Cs|||}d|jvr>t|jjdkr>tj|jj|jjd}t|jj}d}d|jvrlt|jj }t|}|j}| |\}} |dkr|t |||j |j dfdti| } n |t |||j |j dfi| } || _ | S)Nrrr)rPrrr)rarBprodrrrvrHrZrQrPrq_unconvert_indexrLrr5) rrrrrorZrPrrrrrFrFrGr~ s:      zGenericFixed.read_index_noder)rrcCsJtd|j}|j|j||t|j|}t|j|j _ |j |j _ dS)zwrite a 0-len arrayrUN) rBrrGr create_arrayrr`rNrrarvr)rrrZarrrrFrFrGwrite_array_empty s  zGenericFixed.write_array_emptyrwz Index | Noner~)rrrrc CsJt|dd}||jvr&|j|j||jdk}d}t|jrFtd|s^t|dr^|j }d}d}|j durt t  t j|j}Wdn1s0Y|dur|s|jj|j|||j|j d}||dd<n |||nL|jjtjkr`tj|dd} |rn,| d krn t| ||f} tj| ttd |j|j|t } | |nt |jr|j!|j||"d d t#|j|j$_%nt&|jr|j!|j||j't#|j|} t(|j)| j$_)d | j$_%n\t*|jr|j!|j||"d d t#|j|j$_%n&|r$|||n|j!|j|||t#|j|j$_+dS)NT)Z extract_numpyrFz]Cannot store a category dtype in a HDF5 dataset that uses format="fixed". Use format="table".rw)rZskipnarrrr$r'),r5rrrr rrrrrwrrrruZAtomZ from_dtypeZ create_carrayrrrrBZobject_r infer_dtypeperformance_docrrrrZcreate_vlarray ObjectAtomrrrviewr`rarvrasi8rrr#ru) rrrrrZ empty_arrayrurca inferred_typerZvlarrrrFrFrGr sf       .        zGenericFixed.write_array)NN)NN)NN)NN)N)rcrdrerr'r*rdrrfrbrrerhrqrrrrrVrWrryrrrr}r~rrrFrFrFrGr` s2   .#& r`csNeZdZUdZdgZded<eddZd dddd d Zfd d Z Z S)rr}rPr c Cs.zt|jjfWSttfy(YdS0dSrI)r^rrNrrrrrFrFrGr4 szSeriesFixed.shapeNrxr\cCs<||||jd||d}|jd||d}t|||jdS)Nrr\rN)rrP)rrrryr+rP)rr_rrrrrNrFrFrGr; s zSeriesFixed.readc s<tj|fi||d|j|d||j|j_dS)NrrN)rrrrrrPrrtrrFrGrG s zSeriesFixed.write)NNNN) rcrdrerCrbrrrrrr>rFrFrrGr. s   rcsReZdZUddgZded<eddddZdd d d d d Zfd dZZ S)BlockManagerFixedrGnblocksrRz Shape | NonercCsz|j}d}t|jD]8}t|jd|d}t|dd}|dur||d7}q|jj}t|dd}|durt|d|d}ng}|||WStyYdS0dS)Nrblock_itemsrrU) rGrFrr`rZ block0_valuesr\rrr)rrGrrrrrFrFrGrS s"    zBlockManagerFixed.shapeNrxr\cCs||||d}g}t|jD]<}||kr<||fnd\}} |jd||| d} || q(|d} g} t|jD]Z}|d|d} |jd|d|| d}| | | }t |j ||dd }| |q|t | dkrt | dd }|j| d d }|St |d|dd S) Nr)NNrr\rrr9rUrrrAF)rrp)rrrFZ_get_block_manager_axisrFrGrrrryrLr&rwr^r-rS)rr_rrrZ select_axisr9rrraxrdfs blk_itemsrNdfoutrFrFrGrn s(    zBlockManagerFixed.readc stj|fi|t|jtr*|d}|j}|s@|}|j|j _t |j D]0\}}|dkrr|j srt d|d||qTt|j|j _t |jD]D\}}|j|j}|jd|d|j|d|d|d|qdS)Nrrz/Columns index has to be unique for fixed formatrr9)rr)rrrA_mgrr7 _as_managerZis_consolidatedZ consolidaterGrrr9Z is_uniquerrr^blocksrrrMmgr_locsrrN)rrrrorrblkrrrFrGr s    zBlockManagerFixed.write)NNNN) rcrdrerbrrrrrr>rFrFrrGrN s %rc@seZdZdZeZdS)rr~N)rcrdrerCr&rFrFrFrFrGr srcseZdZUdZdZdZded<ded<dZded <d Zd ed <d ed<ded<ded<ded<ded<dddddfdd Z e ddddZ ddd d!Z dd"d#d$Z d%d&Ze d'dd(d)Zd*d+d,d-d.Ze d/dd0d1Ze d'dd2d3Ze d4d5Ze d6d7Ze d8d9Ze d:d;Ze dd?Ze d'dd@dAZe dBdCZdDddEdFZdGdHZdIddJdKZdddLdMdNZddOdPdQdRZddSdTdUZ dVdWZ!dXdYZ"ddZd[Z#d\d]Z$e%d^d_Z&dd`dadbdcZ'ddddddedfdgdhZ(e)d'didjdkZ*dldmZ+ddnd'dodpdqZ,e-dnd'drdsdtZ.ddudvdwdxZ/ddd'dddDdydzd{Z0dddddd|d}d~Z1dddddddddZ2Z3S)raa represent a table: facilitate read/write of various types of tables Attrs in Table Node ------------------- These are attributes that are store in the main table node, they are necessary to recreate these tables when read back in. index_axes : a list of tuples of the (original indexing axis and index column) non_index_axes: a list of tuples of the (original index axis and columns on a non-indexing axis) values_axes : a list of the columns which comprise the data of this table data_columns : a list of the columns that we are allowing indexing (these become single columns in values_axes) nan_rep : the string to use for nan representations for string objects levels : the names of levels metadata : the names of the metadata columns Z wide_tablerkrNrDrzrUzint | list[Hashable]rTzlist[IndexCol] index_axeszlist[tuple[int, Any]]rz list[DataCol] values_axesr\rrr;rtNrorr>)rrrc sPtj||||d|pg|_|p$g|_|p.g|_|p8g|_| pBi|_| |_dS)Nr)rrrrrrrtr) rrrrLrrrrrrtrrrFrGr s      zTable.__init__rcCs|jddS)N_r)rzrrrFrFrGtable_type_short szTable.table_type_shortc Cs|t|jrd|jnd}d|d}d}|jrZddd|jD}d|d}dd d|jD}|jd |d |jd |j d |j d|d|d S)rNrrcz,dc->[rrMcSsg|] }t|qSrFrNrrFrFrGr[ rz"Table.__repr__..rPcSsg|] }|jqSrFrOrmrFrFrGr[ rrQz (typ->z,nrows->z,ncols->z ,indexers->[rR) rr^rrrJrIrr]rrncols)rZjdcrUverZjverZ jindex_axesrFrFrGr s(  zTable.__repr__)rcCs"|jD]}||jkr|SqdS)zreturn the axis for cN)r9rP)rrrvrFrFrGr s   zTable.__getitem__c Cs|dur dS|j|jkr2td|jd|jddD]~}t||d}t||d}||kr6t|D]4\}}||}||krbtd|d|d|dqbtd|d|d|dq6dS) z"validate against an existing tableNz'incompatible table_type with existing [rr)rrrzinvalid combination of [z] on appending data [z] vs current table [)rzrr`rrr4)rrrsvovrsaxZoaxrFrFrGrY s:   zTable.validaterzcCs t|jtS)z@the levels attribute is 1 or a list in the case of a multi-index)rArr\rrFrFrGis_multi_index+ szTable.is_multi_indexrwz tuple[DataFrame, list[Hashable]])rrc Cs`t|jj}z |}Wn.tyH}ztd|WYd}~n d}~00t|tsXJ||fS)ze validate that we can store the multi-index; reset and return the new object zBduplicate names/columns in the multi-index when storing as a tableN)r5Zfill_missing_namesrrZ reset_indexrrAr&)rrrZ reset_objr8rFrFrGvalidate_multiindex0 s zTable.validate_multiindexrRcCstdd|jDS)z-based on our axes, compute the expected nrowscSsg|]}|jjdqSr)rrrWrrFrFrGr[D rz(Table.nrows_expected..)rBrrrrFrFrGnrows_expectedA szTable.nrows_expectedcCs d|jvS)zhas this table been createdrkrrrFrFrGrF szTable.is_existscCst|jddSNrkr`rrrFrFrGrXK szTable.storablecCs|jS)z,return the table group (this is my storable))rXrrFrFrGrkO sz Table.tablecCs|jjSrI)rkrrrFrFrGrT sz Table.dtypecCs|jjSrIrrrFrFrGrX szTable.descriptioncCst|j|jSrI)r&r'rrrrFrFrGr9\ sz Table.axescCstdd|jDS)z.the number of total columns in the values axescss|]}t|jVqdSrI)r^rNrmrFrFrGr@c rzTable.ncols..)sumrrrFrFrGr` sz Table.ncolscCsdSrrFrrFrFrG is_transposede szTable.is_transposedcCs(ttdd|jDdd|jDS)z@return a tuple of my permutated axes, non_indexable at the frontcSsg|]}t|dqSrrLrmrFrFrGr[n rz*Table.data_orientation..cSsg|]}t|jqSrF)rRrrmrFrFrGr[o r)r]r&r'rrrrFrFrGdata_orientationi s zTable.data_orientationzdict[str, Any]csRdddddjD}fddjD}fddjD}t|||S)z.csg|]\}}|dfqSrIrF)rWrrN) axis_namesrFrGr[z rcs&g|]}|jtjvr|j|fqSrF)rPrErrr,rrFrGr[{ s)rrrr;)rZd1Zd2Zd3rF)rrrG queryabless s  zTable.queryablescCsdd|jDS)zreturn a list of my index colscSsg|]}|j|jfqSrF)rrrrFrFrGr[ rz$Table.index_cols..rrrFrFrG index_cols szTable.index_colsrcCsdd|jDS)zreturn a list of my values colscSsg|] }|jqSrFrrrFrFrGr[ rz%Table.values_cols..)rrrFrFrG values_cols szTable.values_colsrcCs|jj}|d|dS)z)return the metadata pathname for this keyz/meta/z/metarTrrFrFrG_get_metadata_path szTable._get_metadata_pathr)rrNcCs,|jj||t|d|j|j|jddS)z Write out a metadata array to the key as a fixed-format Series. Parameters ---------- key : str values : ndarray rk)rrLrrN)rrrr+rLrr)rrrNrFrFrGr s zTable.write_metadatarcCs0tt|jdd|ddur,|j||SdS)z'return the meta data array for this keyrN)r`rrrrrrFrFrGr szTable.read_metadatacCspt|j|j_||j_||j_|j|j_|j|j_|j|j_|j|j_|j |j_ |j |j_ |j |j_ dS)zset our table type & indexablesN) rNrzrrrrrrrLrrrtrrFrFrGrV s        zTable.set_attrscCst|jddpg|_t|jddp$g|_t|jddp8i|_t|jdd|_tt|jdd|_tt|jdd|_ t|jd dpg|_ d d |j D|_ d d |j D|_ dS) rsrNrrtrrLrrorcSsg|]}|jr|qSrFrrmrFrFrGr[ rz#Table.get_attrs..cSsg|]}|js|qSrFrrmrFrFrGr[ r)r`rrrrtrrMrLrHrr indexablesrrrrFrFrGrW szTable.get_attrscCs\|durX|jddkrX|jddkrX|jddkrXtddd|jD}t|tdS) rZNrrUrGrHrMcSsg|] }t|qSrFrrrFrFrGr[ rz*Table.validate_version..)rIincompatibility_docrrrrg)rr_rrFrFrGr[ s*zTable.validate_versioncCsR|dur dSt|tsdS|}|D]&}|dkr4q&||vr&td|dq&dS)z validate the min_itemsize doesn't contain items that are not in the axes this needs data_columns to be defined NrNzmin_itemsize has the key [z%] which is not an axis or data_column)rAr;rr)rrqrrFrFrGvalidate_min_itemsize s  zTable.validate_min_itemsizec sg}jjjtjjD]j\}\}}t|}|}|durJdnd}|d}t|d} t|||| |j||d} || qt j t |fdd| fddtjj D|S) z/create/cache the indexables if they don't existNrr)rPrrrZrrkrrc st|tsJt}|vrt}t|}t|j}t|dd}t|dd}t|}|}t|dd} |||||||j | ||d } | S)Nrrr) rPrrNrZrrrkrrr) rArNrr?r`_maybe_adjust_namerIrrrk) rrklassradj_namerNrrZmdrr)base_posrUdescr table_attrsrFrGrls0    zTable.indexables..fcsg|]\}}||qSrFrF)rWrr)rlrFrGr['rz$Table.indexables..)rrkrrrr`rrrrErr^rIr) r _indexablesrrrPrrrrrZ index_colrF)rrUrrlrrrGr s2      % zTable.indexablesryr c CsV|s dS|durdS|dus(|dur8dd|jD}t|ttfsL|g}i}|dur`||d<|durp||d<|j}|D]}t|j|d}|dur"|jr|j }|j } |j } |dur| |kr| n| |d<|dur| |kr| n| |d<|jsP|j drtd |jfi|qz||jd d vrztd |d |d|dqzdS)aZ Create a pytables index on the specified columns. Parameters ---------- columns : None, bool, or listlike[str] Indicate which columns to create an index on. * False : Do not create any indexes. * True : Create indexes on all columns. * None : Create indexes on all columns. * listlike : Create indexes on the given columns. optlevel : int or None, default None Optimization level, if None, pytables defaults to 6. kind : str or None, default None Kind of index, if None, pytables defaults to "medium". Raises ------ TypeError if trying to create an index on a complex-type column. Notes ----- Cannot index Time64Col or ComplexCol. Pytables must be >= 3.0. NFTcSsg|]}|jr|jqSrF)rrrmrFrFrGr[Nrz&Table.create_index..rYrZcomplexzColumns containing complex values can be stored but cannot be indexed when using table format. Either use fixed format, set index=False, or do not include the columns containing complex values to data_columns when initializing the table.rrUzcolumn z/ is not a data_column. In order to read column z: you must reload the dataframe into HDFStore and include z with the data_columns argument.)rr9rAr]r\rkr`r?rlrrYrZZ remove_indexrrhrr[rrr) rrrYrZkwrkrrCrZ cur_optlevelZcur_kindrFrFrGr[+sR   zTable.create_indexrxz!list[tuple[ArrayLike, ArrayLike]]rrrc CsZt||||d}|}g}|jD]2}||j|j||j|j|jd}| |q"|S)a Create the axes sniffed from the table. Parameters ---------- where : ??? start : int or None, default None stop : int or None, default None Returns ------- List[Tuple[index_values, column_values]] r r3) Selectionrr9rrtrrrLrr) rr_rr selectionrNrrvresrFrFrG _read_axess   zTable._read_axesrucCs|S)zreturn the data for this objrFrrrurFrFrG get_objectszTable.get_objectcst|s gS|d\}|j|i}|ddkrL|rLtd|d||dur^t}n |durjg}t|trt|t|}|fdd | Dfd d |DS) zd take the input data_columns and min_itemize and create a data columns spec rrr)z"cannot use a multi-index on axis [z] with data_columns TNcs g|]}|dkr|vr|qSrrFr)existing_data_columnsrFrGr[sz/Table.validate_data_columns..csg|]}|vr|qSrFrF)rWr) axis_labelsrFrGr[r) r^rtrrr\rAr;rErIr)rrrrrrtrF)rrrGvalidate_data_columnss.     zTable.validate_data_columnsr&)rrYc/stts,|jj}td|dtddur:dgfddD|rzd}d d|jDt|j }|j }nd }|j } |j d ksJt |j d krtd g} |durd}fdddDd} j| } t| } |r|j|j|j|| ||-| |d }.t?|drl|j@|._@|.A||r|r|.B||.S)a0 Create and return the axes. Parameters ---------- axes: list or None The names or numbers of the axes to create. obj : DataFrame The object to create axes on. validate: bool, default True Whether to validate the obj against an existing object already written. nan_rep : A value to use for string column nan_rep. data_columns : List[str], True, or None, default None Specify the columns that we want to create to allow indexing on. * True : Use all available columns. * None : Use no columns. * List[str] : Use the specified columns. min_itemsize: Dict[str, int] or None, default None The min itemsize for a column in bytes. z/cannot properly create the storer for: [group->rwrNrcsg|]}|qSrF)_get_axis_numberrm)rrFrGr[rz&Table._create_axes..TcSsg|] }|jqSrFrArmrFrFrGr[rFrHrUz.get_blk_items..)rrrFrrG get_blk_itemssz2Table._get_blocks_and_items..get_blk_itemsrrAcSs"i|]\}}t|||fqSrF)r]tolist)rWbrrFrFrGrDsz/Table._get_blocks_and_items..rcSsg|] }t|qSrFrO)rWitemrFrFrGr[rz/Table._get_blocks_and_items..z+cannot match existing table structure for [z] on appending data)rArr7rr r8r\rr^r(rJrSrIr(r]rNrRrrrrr)r~rrrrrrrrrrZ new_labelsrZby_itemsZ new_blocksZ new_blk_itemsZearrrr8ZjitemsrFrFrGrsN          zTable._get_blocks_and_itemsr)rc s|durt|}|durNjrNtjts.JjD]}||vr4|d|q4jD]\}}t|||qT|jdur|jD]$\}}fdd} | ||qS)zprocess axes filtersNrcsjD]}|}|}|dus*J||krfjrH|tj}||}j|d|S||vrtt |j }t|}t t rd|}||}j|d|Sqt d|ddS)NrArUzcannot find the field [z] for filtering!)Z _AXIS_ORDERSr _get_axisrunionr(rrQr6r`rNrAr&r)fieldfiltrZ axis_numberZ axis_valuesZtakersrNroprrFrGprocess_filter s"       z*Table.process_axes..process_filter) r\rrArinsertrrfilterr) rrrrrrlabelsrrrrFrrG process_axess  ! zTable.process_axes)rrr:rcCs|durt|jd}d|d}dd|jD|d<|rj|durH|jpFd}tj|||pZ|jd }||d <n|jdur~|j|d <|S) z:create the description of the table from the axes & valuesNi'rk)rPr:cSsi|]}|j|jqSrF)rrrmrFrFrGrD>rz,Table.create_description..r )rrrr)maxrr9rrurrr)rrrrr:r<rrFrFrGcreate_description/s       zTable.create_descriptionr\c Cs|||sdSt||||d}|}|jdur|jD]D\}}}|j|||dd} ||| j |||j }qBt |S)zf select coordinates (row numbers) from a table; return the coordinates object Fr NrUr\) r[rr select_coordsrrrrrilocrNr() rr_rrrZcoordsrrrrorFrFrGrNs   zTable.read_coordinatesrcCs||sdS|dur$td|jD]z}||jkr*|jsNtd|dt|jj |}| |j |j ||||j |j|jd}tt|d|j|dSq*td|d dS) zj return a single column from the table, generally only indexables are interesting FNz4read_column does not currently accept a where clausezcolumn [z=] can not be extracted individually; it is not data indexabler3rUrOz] not found in the table)r[rrr9rPrrr`rkr?rrtrrrLrr+rrr)rrr_rrrvrZ col_valuesrFrFrGrhs*      zTable.read_column)NroNNNNNN)N)NNN)NN)TNNN)N)NNN)NNN)4rcrdrerrCrDrrr)rrrrrrYrrrrrXrkrrr9rrrrrrrrrrVrWr[rrrr[rr=rrr staticmethodrrrrrr>rFrFrrGr s   "         KU"*k>: rc@s.eZdZdZdZd dddddZdd ZdS) rz a write-once read-many table: this format DOES NOT ALLOW appending to a table. writing is a one-time operation the data are stored in a format that allows for searching the data on disk rNrxr\cCs tddS)z[ read the indices and the indexing array, calculate offset rows and return z!WORMTable needs to implement readNr]r^rFrFrGrs zWORMTable.readcKs tddS)z write in a format that we can search later on (but cannot append to): write out the indices and the values using _write_array (e.g. a CArray) create an indexing table so that we can search z"WORMTable needs to implement writeNr]r_rFrFrGrszWORMTable.write)NNNN)rcrdrerrzrrrFrFrFrGrs rc @sVeZdZdZdZdddZddd d d d Zd ddddddZddddddZdS)r(support the new appendable table formatsZ appendableNFTcCs|s|jr|j|jd|j||||| | d}|jD] }|q6|js|j|||| d}|| |d<|jj |jfi||j |j _ |jD]}| ||q|j || ddS)Nrk)r9rrYrrr)rrrr:r-)r)rrrrrr9rrrVZ create_tablertrr write_data)rrr9rrrrrrr:rrrr-rkrvoptionsrFrFrGrs4     zAppendableTable.writerxrz)rrcs|jj}|j}g}|rT|jD]6}t|jjdd}t|tj r| |j dddqt |r|d}|ddD] }||@}qp| }nd}dd |jD} t | } | dksJ| d d |jD} d d | D} g} t| D]6\} }|f|j|| | j}| | | |q|dur$d }tjt|||jd }||d}t|D]x} | |t| d||kr|q|j|fdd | D|dur|ndfdd | DdqNdS)z` we form the data into a 2-d including indexes,values,mask write chunk-by-chunk rrAu1Fr1rUNcSsg|] }|jqSrF)rrmrFrFrGr[rz.AppendableTable.write_data..cSsg|] }|qSrF)rrmrFrFrGr[rc Ss,g|]$}|tt|j|jdqSr)Z transposerBZrollrrGr,rFrFrGr[rrr)csg|]}|qSrFrFrmZend_iZstart_irFrGr[)rcsg|]}|qSrFrFr,rrFrGr[+r)indexesr<rN)rrrrr.ror=rArBrrr8r^r6rrrreshaperrrFwrite_data_chunk)rrrrrmasksrvr<mrnindexesrNbvaluesrrCZ new_shaperowschunksrFrrGr sL        zAppendableTable.write_datarzlist[np.ndarray]znp.ndarray | None)rrr<rNc Cs|D]}t|jsdSq|djd}|t|krFtj||jd}|jj}t|}t|D]\} } | ||| <q^t|D]\} }|||| |<q||dur|j t dd} | s|| }t|r|j ||j dS)z Parameters ---------- rows : an empty memory space where we are putting the chunk indexes : an array of the indexes mask : an array of the masks values : an array of the values Nrr)Fr1)rBrrr^rrrrr6r8rzr=rkrr) rrrr<rNrCrrrrrrrFrFrGr.s&   z AppendableTable.write_data_chunkr\cCsb|dust|sf|dur:|dur:|j}|jj|jddn(|durH|j}|jj||d}|j|S|srdS|j}t ||||d}| }t | }t|} | r^| } t| | dkj} t| sdg} | d| kr| | | ddkr| dd| } t| D]@} |t| | }|j||jd||jddd| } q|j| S)NTr1r\rUrr/)r^rrrrrkZ remove_rowsrrrrr+Z sort_valuesdiffr\rrrrRreversedrMrF)rr_rrrrkrrNZ sorted_serieslnrrZpgrrrFrFrGr7ZsD       zAppendableTable.delete) NFNNNNNNFNNT)F)NNN) rcrdrerrzrr rr7rFrFrFrGrs$ <;,rc@s`eZdZUdZdZdZdZeZde d<e ddd d Z e dd d d Z ddddddZdS)rr r{rrHrErFrzrcCs|jdjdkS)NrrU)rrrrFrFrGrsz"AppendableFrameTable.is_transposedrcCs|r |j}|S)zthese are written transposed)rwrrFrFrGrszAppendableFrameTable.get_objectNrxr\cs0|sdSj|||d}tjrHjjddini}fddtjD}t|dkstJ|d}||d} g} tjD]N\} } | j vrq|| \} }|ddkrt | }n t | }|d}|dur|j |d d jr |}|}t | t| d dd }n|j}t | t| d dd }|}|jdkrlt|tjrl|d|jdf}t|tjrt|j||d }n.t|t rt|||d }ntj|g||d }|j|jksJ|j|jf| |qt| dkr| d}n t| dd}t|||d}j|||d}|S)Nr rcs"g|]\}}|jdur|qSrr)rWrrrrFrGr[rz-AppendableFrameTable.read..rUrr)rTZinplacerPrOrrA)rr) r[rrr^rrtrrr9rr(r) from_tuples set_namesrr`rwrGrArBrrrr&Z _from_arraysZdtypesrr=rr-rr)rr_rrrrortZindsindrframesrrvZ index_valsrr?rrNZindex_Zcols_rrrFrrGrsZ        "   zAppendableFrameTable.read)NNNN)rcrdrerrCrzrGr&rFrrrr=rrrFrFrFrGrs  rcsneZdZdZdZdZdZeZe ddddZ e dd d d Z dfd d Z dddddfdd ZZS)rr rrrHrzrcCsdSrrFrrFrFrGrsz#AppendableSeriesTable.is_transposedrcCs|SrIrFrrFrFrGrsz AppendableSeriesTable.get_objectNc s<t|ts|jpd}||}tjf||jd|S)+we are going to write this as a frame tablerNrr)rAr&rPZto_framerrrr)rrrrrPrrFrGr s   zAppendableSeriesTable.writerxr+rcs|j}|durB|rBt|jts"J|jD]}||vr(|d|q(tj||||d}|rj|j|jdd|jdddf}|j dkrd|_ |S)NrrTrrN) rrArr\rrr set_indexrrP)rr_rrrrrrErrFrGrs   zAppendableSeriesTable.read)N)NNNN)rcrdrerrCrzrGr+rFrrr=rrrr>rFrFrrGrs rcs(eZdZdZdZdZfddZZS)rr rrc s^|jpd}||\}|_t|jts*Jt|j}||t||_tj fd|i|S)r rNr) rPrrrAr\rr(rrr)rrrrPZnewobjr?rrFrGr2s    z AppendableMultiSeriesTable.write)rcrdrerrCrzrr>rFrFrrGr,src@sdeZdZUdZdZdZdZeZde d<e ddd d Z e d d Z d dZ eddZddZdS)rz:a table that read/writes the generic pytables table formatr{r|rHzlist[Hashable]rrNrcCs|jSrI)rCrrFrFrGr]FszGenericTable.pandas_typecCst|jddp|jSrrrrFrFrGrXJszGenericTable.storablecCsLg|_d|_g|_dd|jD|_dd|jD|_dd|jD|_dS)rsNcSsg|]}|jr|qSrFrrmrFrFrGr[Trz*GenericTable.get_attrs..cSsg|]}|js|qSrFrrmrFrFrGr[UrcSsg|] }|jqSrFrOrmrFrFrGr[Vr)rrrrrrrrrFrFrGrWNs zGenericTable.get_attrsc Cs|j}|d}|durdnd}tdd|j||d}|g}t|jD]^\}}t|tsZJt||}||}|durzdnd}t |||g||j||d} | | qD|S)z0create the indexables from the table descriptionrNrr)rPrrkrr)rPrrNrrkrr) rrrrkrZ_v_namesrArNr`rAr) rr<rrrrrrrrUrFrFrGrXs.     zGenericTable.indexablescKs tddS)Nz cannot write on an generic tabler]r_rFrFrGr{szGenericTable.writeN)rcrdrerrCrzrGr&rFrrr]rXrWrrrrFrFrFrGr=s    "rcs`eZdZdZdZeZdZe dZ e ddddZ dfd d Z dd d d fdd ZZS)rza frame with a multi-indexrrHz ^level_\d+$rNrcCsdS)NZappendable_multirFrrFrFrGrsz*AppendableMultiFrameTable.table_type_shortNc sx|durg}n|dur |j}||\}|_t|jts@J|jD]}||vrF|d|qFtjf||d|S)NTrr!) rrrrrAr\rrr)rrrrrrrFrGrs  zAppendableMultiFrameTable.writerxr\csDtj||||d}|j}|jfdd|jjD|_|S)Nrcs g|]}j|rdn|qSrI) _re_levelssearch)rWrPrrFrGr[rz2AppendableMultiFrameTable.read..)rrr"rrrr)rr_rrrrrrrGrs  zAppendableMultiFrameTable.read)N)NNNN)rcrdrerrzr&rFrGrecompiler#rrrrr>rFrFrrGrs rr&r()rrrrcCs||}t|}|dur"t|}|dus4||rB||rB|St|}|durlt|j|dd}||stddg|j}|||<|jt|}|S)NF)sort) rr6equalsuniquerPslicerGrQr])rrrrrZslicerrFrFrGrs   rrz str | tzinfo)rrcCst|}|S)z+for a tz-aware type, return an encoded zone)rZ get_timezone)rzonerFrFrGrs rznp.ndarray | Indexzstr | tzinfo | Noneznp.ndarray | DatetimeIndex)rNrr&rcCst|tr"|jdus"|j|ks"J|durtt|trB|j}|j}n d}|}t|}t||d}|d|}n|rt j |dd}|S)a coerce the values to a DatetimeIndex if tz is set preserve the input shape if possible Parameters ---------- values : ndarray or Index tz : str or tzinfo coerce : if we do not have a passed timezone, coerce to M8[ns] ndarray NrOrkM8[ns]r)) rAr'rrPrr6rHrmrnrBr4)rNrr&rPrFrFrGrs   r)rPrrLrrc Cstt|tsJ|j}t|\}}t|}t|}t|tsFt|j rlt ||||t |ddt |dd|dSt|t r~t dtj|dd} t|} | dkrtjdd | Dtjd }t ||dt|d S| d krt| ||}|j j} t ||d t| |d S| d vr$t |||||dSt|tjr>|j tksBJ|dksTJ|t}t |||||d SdS)Nrr)rNrZrrrrzMultiIndex not supported here!FrrcSsg|] }|qSrF) toordinalr,rFrFrGr[rz"_convert_index..r))rr)integerZfloating)rNrZrrr5)rArNrPrrr?rr/r$rrr`r)rrrrBr4Zint32ruZ Time32Col_convert_string_arrayrrrr5r) rPrrLrrr;r rZrrrNrrFrFrGrsT            r)rZrLrrcCs|dkrt|}n|dkr$t|}n|dkrvztjdd|Dtd}Wqtyrtjdd|Dtd}Yq0nT|dvrt|}n@|d vrt|d||d }n&|d krt|d }ntd ||S)Nr$r'rcSsg|]}t|qSrFr*r,rFrFrGr[6rz$_unconvert_index..r)cSsg|]}t|qSrFr-r,rFrFrGr[8r)r.floatrr3r5rzunrecognized index type )r'r,rBr4r5rr:)rorZrLrrrFrFrGr-s&     rrr)rPrrcCs|jtkr|Sttj|}|jj}tj|dd} | dkrBtdn&| dkrTtdn| dksh|dksh|St |} | } || | <tj| dd} | dkrt | j d D]V} | | } tj| dd} | dkrt || kr|| nd | }td |d | d qt| ||| j }|j}t|trBt||p>|dp>d }t|pLd |}|dur~||}|dur~||kr~|}|jd|dd}|S)NFrrz+[date] is not implemented as a table columnraz>too many timezones in this block, create separate data columnsrr5rzNo.zCannot serialize the column [z2] because its data contents are not [string] but [z] object dtyperNz|Sr1)rr5r rBrrPrrrr.rprFrr^r/rrrAr;rRrrrr8)rPrrrrrLrrr rr<rorrZerror_column_labelrrZecirFrFrGrFsN       rr)rorLrrcCs\t|r(t|j||j|j}t|}t dt |}t j |d|d}|S)a Take a string-like that is object dtype and coerce to a fixed size string type. Parameters ---------- data : np.ndarray[object] encoding : str errors : str Handler for encoding errors. Returns ------- np.ndarray[fixed-length-string] rUSr))r^r+r6rNencoder9rrrr libwritersmax_len_string_arrayrBr4)rorLrensuredrrFrFrGr/s r/cCs|j}tj|td}t|rvtt|}d|}t |dt r^t |j j ||dj}n|j|ddjtdd}|durd}t||||S) a* Inverse of _convert_string_array. Parameters ---------- data : np.ndarray[fixed-length-string] nan_rep : the storage repr of NaN encoding : str errors : str Handler for encoding errors. Returns ------- np.ndarray[object] Decoded data. r)Ur)rFr1Nr)rrBr4r6r5r^r3r4rrArpr+rNrDr9r8Z!string_array_replace_from_nan_repr)rorrLrrrrrFrFrGr:s  r:)rNrrLrcCs6t|tsJt|t|r2t|||}||}|SrI)rArNr _need_convert_get_converter)rNrrLrconvrFrFrGrs  rrZrLrcs8|dkrddS|dkr&fddStd|dS)Nr$cSstj|ddS)Nr,r))rBr4rrFrFrGrrz _get_converter..rcst|ddS)Nr3)r:r;rrFrGrsz invalid kind )rr:rFrrGr8s r8rcCs|dvr dSdS)N)r$rTFrFr rFrFrGr7sr7z Sequence[int])rPrIrcCslt|tst|dkrtd|ddkrh|ddkrh|ddkrhtd|}|rh|d}d|}|S) z Prior to 0.10.1, we named values blocks like: values_block_0 an the name values_0, adjust the given name if necessary. Parameters ---------- name : str version : Tuple[int, int, int] Returns ------- str z6Version is incorrect, expected sequence of 3 integers.rrUrGrHzvalues_block_(\d+)Zvalues_)rArNr^rr%r$r)rPrIrgrprFrFrGrs$   r) dtype_strrcCst|}|ds|dr"d}n|dr2d}n|drBd}n|dsV|dr\d}nn|drld}n^|d r|d }nN|d rd }n>|d rd }n.|d rd}n|dkrd}ntd|d|S)zA Find the "kind" string describing the given dtype name. rrpr0rrRrr.r$ timedeltar'rzrrr5zcannot interpret dtype of [r)rHrhr)r>rZrFrFrGr s.       rrcCsbt|tr|j}|jjdd}|jjdvr@t| d}nt|t rP|j }t|}||fS)zJ Convert the passed data into a storable form and a dtype string. rPr)rMr) rAr0rrrPrrZrBr4rr*r)ror rFrFrGr*s    rc@s<eZdZdZdddddddZdd Zd d Zd d ZdS)rz Carries out a selection operation on a tables.Table object. Parameters ---------- table : a Table object where : list of Terms (or convertible to) start, stop: indices to start and/or stop selection Nrrx)rkrrcCsh||_||_||_||_d|_d|_d|_d|_t|r.t t t j |dd}|dksd|dkrt |}|jt jkr|j|j}}|durd}|dur|jj}t ||||_nVt|jjt jr|jdur||jks|jdur||jkrt d||_Wdn1s$0Y|jdurd|||_|jdurd|j\|_|_dS)NFrr.booleanrz3where must have index locations >= start and < stop)rkr_rr conditionrZtermsr%r!rrrrrBr4rZbool_rr issubclassrr.r7generateevaluate)rrkr_rrinferredrFrFrGrLsD     &   zSelection.__init__c Cs|dur dS|j}zt|||jjdWSty|}z:d|}td|d|d}t||WYd}~n d}~00dS)z'where can be a : dict,list,tuple,stringN)rrLrz- The passed where expression: a* contains an invalid variable reference all of the variable references must be a reference to an axis (e.g. 'index' or 'columns'), or a data_column The currently defined references are: z ) rkrr3rL NameErrorrrrr)rr_rr8ZqkeysrrFrFrGrD{s  zSelection.generatecCsX|jdur(|jjj|j|j|jdS|jdurB|jj|jS|jjj|j|jdS)( generate the selection Nr\) rBrkZ read_whererrrr%rrrrFrFrGrs  zSelection.selectcCs|j|j}}|jj}|dur$d}n|dkr4||7}|durB|}n|dkrR||7}|jdurx|jjj|j||ddS|jdur|jSt ||S)rHNrT)rrr') rrrkrrBZget_where_listrr%rBr)rrrrrFrFrGrs"  zSelection.select_coords)NNN)rcrdrerrrDrrrFrFrFrGr@s/ r) rvNNFNTNNNNror@) NrroNNNNFN)N)F)r __future__r contextlibrrprarrr&rr%textwraprtypingrrr r r r r rnumpyrBZpandas._configrrZ pandas._libsrrr3Zpandas._libs.tslibsrZpandas._typingrrrZpandas.compat._optionalrZpandas.compat.pickle_compatrZ pandas.errorsrZpandas.util._decoratorsrZpandas.util._exceptionsrZpandas.core.dtypes.commonrrrrrr r!r"r#r$Zpandas.core.dtypes.missingr%rr&r'r(r)r*r+r,r-r.Zpandas.core.apir/Zpandas.core.arraysr0r1r2Zpandas.core.commoncorecommonr5Z pandas.core.computation.pytablesr3r4Zpandas.core.constructionr5Zpandas.core.indexes.apir6Zpandas.core.internalsr7r8Zpandas.io.commonr9Zpandas.io.formats.printingr:r;rqr<r=r>r?rSrJrHrMrQrVr`r4rarfWarningrgrrhrriZ duplicate_docrrurHZ dropna_docZ format_docZ config_prefixZregister_optionZis_boolZis_one_of_factoryrprtrurrrrrrrrr?rArBr`rrrrrrrrrrrrrrrrrr/r:rr8r7rrrrrFrFrFrGs0   $       0 ,         &*:Np'1d _ifd1B+ (=K' !