a < b@sdZddlmZddlmZddlZddlmZmZddl Z ddl Z ddl Z ddl m Z ddlmZmZmZmZmZmZmZddlZddlZddlmZmZdd lmZmZdd l m!Z!dd l"m#Z#m$Z$m%Z%dd l&m'Z'dd l(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:ddl;mZ>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFddlGmHZHddlImJZJmKZKmLZLddlMmNmOZPddlQmRZRmSZSddlTmUZUddlVmWZWddlXmYZYmZZZddl[m\Z\ddl]m^Z^m_Z_erddl`maZambZbmcZcddlXmdZddZedZfd d!Zgd"d#Zhd$d%ZieRZjd&d'd(d)ZkGd*d+d+elZmGd,d-d-elZnGd.d/d/eoZpd0ZqGd1d2d2eoZrd3ZsGd4d5d5eoZtd6Zud7Zvd8d8d9d9d:Zwe>dgiZxd;Zydd?eyej}d@ej|dAdeze~gdBd@Wdn1s 0Ydad?adCdDZddHdIdHdJdKdLdKdLdMdNdOdHdHdPdQdRdSZddHdHdJdJdJdUdVdWZdXdXdLdYdZd[ZGd\d]d]ZGd^d_d_ZGd`dadaZGdbdcdceZGdddedeeZGdfdgdgeZGdhdidieZGdjdkdkZGdldmdmeZGdndodoeZGdpdqdqeZGdrdsdseZGdtdudueZGdvdwdweZGdxdydyeZGdzd{d{eZGd|d}d}eZGd~ddeZGdddeZGdddeZddd&dddddZdddddZddddLddddZdHddHdHdadddZdHdHdHddddZdHdddddZddHdHddddZddHdHddddZddHdHdHdddZdHdHdHdddZdHdLdddZdHddHdddZdHdHdddZddddZGdddZdS)zY High level interface to PyTables for reading and writing pandas data structures to disk ) annotations)suppressN)datetzinfo)dedent) TYPE_CHECKINGAnyCallableHashableLiteralSequencecast)config get_option)libwriters) timezones) ArrayLikeDtypeArgShape)import_optional_dependency) patch_pickle)PerformanceWarning)cache_readonly)find_stack_level) ensure_objectis_categorical_dtypeis_complex_dtypeis_datetime64_dtypeis_datetime64tz_dtypeis_extension_array_dtype is_list_likeis_string_dtypeis_timedelta64_dtypeneeds_i8_conversion)array_equivalent) DataFrame DatetimeIndexIndex MultiIndex PeriodIndexSeriesTimedeltaIndexconcatisna) Int64Index) Categorical DatetimeArray PeriodArray) PyTablesExprmaybe_expression) extract_array) ensure_index) ArrayManager BlockManager)stringify_path)adjoin pprint_thing)ColFileNode)Blockz0.15.2UTF-8cCst|tjr|d}|S)z(if we have bytes, decode them to unicoder@) isinstancenpbytes_decode)srFb/Users/vegardjervell/Documents/master/model/venv/lib/python3.9/site-packages/pandas/io/pytables.py_ensure_decodedus  rHcCs|dur t}|SN)_default_encodingencodingrFrFrG_ensure_encoding|srMcCst|trt|}|S)z Ensure that an index / column name is a str (python 3); otherwise they may be np.string dtype. Non-string dtypes are passed through unchanged. https://github.com/pandas-dev/pandas/issues/13492 )rAstrnamerFrFrG _ensure_strs rQint scope_levelcsV|dt|ttfr*fdd|D}nt|r>t|d}|dusNt|rR|SdS)z Ensure that the where is a Term or a list of Term. This makes sure that we are capturing the scope of variables that are passed create the terms here with a frame_level=2 (we are 2 levels down) cs0g|](}|durt|r(t|ddn|qS)NrUrS)r4Term).0ZtermlevelrFrG sz _ensure_term..rSN)rAlisttupler4rVlen)whererTrFrXrG _ensure_terms   r_c@s eZdZdS)PossibleDataLossErrorN__name__ __module__ __qualname__rFrFrFrGr`sr`c@s eZdZdS)ClosedFileErrorNrarFrFrFrGresrec@s eZdZdS)IncompatibilityWarningNrarFrFrFrGrfsrfz where criteria is being ignored as this version [%s] is too old (or not-defined), read the file in and write it out to a new file to upgrade (with the copy_to method) c@s eZdZdS)AttributeConflictWarningNrarFrFrFrGrgsrgzu the [%s] attribute of the existing index is [%s] which conflicts with the new [%s], resetting the attribute to None c@s eZdZdS)DuplicateWarningNrarFrFrFrGrhsrhz; duplicate entries in table, taking most recently appended z your performance may suffer as PyTables will pickle object types that it cannot map directly to c-types [inferred_type->%s,key->%s] [items->%s] fixedtable)fritrjz; : boolean drop ALL nan rows when appending to a table z~ : format default format writing format, if None, then put will default to 'fixed' and append will default to 'table' zio.hdfZ dropna_tableF)Z validatordefault_format)rirjNcCsLtdurHddl}|att|jjdkaWdn1s>0YtS)Nrstrict) _table_modtablesrAttributeErrorfileZ_FILE_OPEN_POLICY!_table_file_open_policy_is_strict)rprFrFrG_tabless  rtaTrnrNDataFrame | Series int | None str | Noneboolint | dict[str, int] | None bool | None Literal[True] | list[str] | NoneNone)keyvaluemode complevelcomplibappendformatindex min_itemsizedropna data_columnserrorsrLreturnc s|r$ f dd}n f dd}t|}t|trt||||d}||Wdq1s0Yn||dS)z+store this object, close it if we opened itc s|j d S)N)rrrnan_reprrrrLrstore rrrLrrrr~rrrrFrGszto_hdf..c s|j d S)N)rrrrrrrLrputrrrFrGr(s)rrrN)r9rArNHDFStore) path_or_bufr~rrrrrrrrrrrrrLrkrrFrrGto_hdfs   (rr)rrstartstop chunksizec  Ks|dvrtd|d|dur,t|dd}t|trN|jsDtd|} d} ntt|}t|tshtd zt j |} Wnt tfyd} Yn0| st d |d t|f||d | } d } zt|dur| }t|dkrtd|d}|ddD]}t||stdq|j}| j||||||| | dWStt tfyt|tstt| Wdn1s0YYn0dS)a) Read from the store, close it if we opened it. Retrieve pandas object stored in file, optionally based on where criteria. .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- path_or_buf : str, path object, pandas.HDFStore Any valid string path is acceptable. Only supports the local file system, remote URLs and file-like objects are not supported. If you want to pass in a path object, pandas accepts any ``os.PathLike``. Alternatively, pandas accepts an open :class:`pandas.HDFStore` object. key : object, optional The group identifier in the store. Can be omitted if the HDF file contains a single pandas object. mode : {'r', 'r+', 'a'}, default 'r' Mode to use when opening the file. Ignored if path_or_buf is a :class:`pandas.HDFStore`. Default is 'r'. errors : str, default 'strict' Specifies how encoding and decoding errors are to be handled. See the errors argument for :func:`open` for a full list of options. where : list, optional A list of Term (or convertible) objects. start : int, optional Row number to start selection. stop : int, optional Row number to stop selection. columns : list, optional A list of columns names to return. iterator : bool, optional Return an iterator object. chunksize : int, optional Number of rows to include in an iteration when using an iterator. **kwargs Additional keyword arguments passed to HDFStore. Returns ------- item : object The selected object. Return type depends on the object stored. See Also -------- DataFrame.to_hdf : Write a HDF file from a DataFrame. HDFStore : Low-level access to HDF files. Examples -------- >>> df = pd.DataFrame([[1, 1.0, 'a']], columns=['x', 'y', 'z']) # doctest: +SKIP >>> df.to_hdf('./store.h5', 'data') # doctest: +SKIP >>> reread = pd.read_hdf('./store.h5') # doctest: +SKIP )rr+ruzmode zG is not allowed while performing a read. Allowed modes are r, r+ and a.NrUrSz&The HDFStore must be open for reading.Fz5Support for generic buffers has not been implemented.zFile z does not exist)rrTrz]Dataset(s) incompatible with Pandas data types, not table, or no datasets found in HDF5 file.z?key must be provided when HDF5 file contains multiple datasets.)r^rrcolumnsiteratorr auto_close) ValueErrorr_rAris_openOSErrorr9rNNotImplementedErrorospathexists TypeErrorFileNotFoundErrorgroupsr]_is_metadata_of _v_pathnameselectKeyErrorrrqclose)rr~rrr^rrrrrkwargsrrrrZcandidate_only_groupZgroup_to_checkrFrFrGread_hdf?sjO           (rr>)group parent_grouprcCsF|j|jkrdS|}|jdkrB|j}||kr:|jdkr:dS|j}qdS)zDCheck if a given group is a metadata group for a given parent_group.FrUmetaT)Z_v_depthZ _v_parent_v_name)rrcurrentparentrFrFrGrs  rc@seZdZUdZded<ded<ded<ded <ddd ddddZddZeddZeddZ ddddZ ddddZ ddddZ dddd Z ddd!d"d#Zdd$d%d&Zdd$d'd(Zd)d*Zd+d,Zddd.d/d0d1Zd2d3Zd4d5ZeZddd6d7d8Zd9d:Zedd$d;d<Zddd=d>d?Zddd@dAZddddBdCdDZddd d dEdFdGZdddd d dHdIdJZdddKdLdMZdddPd dQdRddddSdTdUZ ddddVdWZ!dddPd dQdXdRddYdZd[Z"dd\d]d^d_Z#ddd d`dadbdcZ$dddeZ%ddgdhZ&ddid!djdkZ'ddld!dmdnZ(ddd ddpdqdrZ)dd$dsdtZ*dudvZ+dddwdxdyZ,dd{dddld|d}d~Z-dddPd dQddddddZ.ddddZ/ddddddZ0ddd!ddZ1d S)raa Dict-like IO interface for storing pandas objects in PyTables. Either Fixed or Table format. .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- path : str File path to HDF5 file. mode : {'a', 'w', 'r', 'r+'}, default 'a' ``'r'`` Read-only; no data can be modified. ``'w'`` Write; a new file is created (an existing file with the same name would be deleted). ``'a'`` Append; an existing file is opened for reading and writing, and if the file does not exist it is created. ``'r+'`` It is similar to ``'a'``, but the file must already exist. complevel : int, 0-9, default None Specifies a compression level for data. A value of 0 or None disables compression. complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib' Specifies the compression library to be used. As of v0.20.2 these additional compressors for Blosc are supported (default if no compressor specified: 'blosc:blosclz'): {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy', 'blosc:zlib', 'blosc:zstd'}. Specifying a compression library which is not available issues a ValueError. fletcher32 : bool, default False If applying compression use the fletcher32 checksum. **kwargs These parameters will be passed to the PyTables open_file method. Examples -------- >>> bar = pd.DataFrame(np.random.randn(10, 4)) >>> store = pd.HDFStore('test.h5') >>> store['foo'] = bar # write to HDF5 >>> bar = store['foo'] # retrieve >>> store.close() **Create or load HDF5 file in-memory** When passing the `driver` option to the PyTables open_file method through **kwargs, the HDF5 file is loaded or created in-memory and will only be written when closed: >>> bar = pd.DataFrame(np.random.randn(10, 4)) >>> store = pd.HDFStore('test.h5', driver='H5FD_CORE') >>> store['foo'] = bar >>> store.close() # only now, data is written to disk z File | None_handlerN_moderR _complevelry _fletcher32ruNFrw)rr fletcher32cKsd|vrtdtd}|dur@||jjvr@td|jjd|durX|durX|jj}t||_|durnd}||_d|_|r|nd|_ ||_ ||_ d|_ |j fd|i|dS) Nrz-format is not a defined argument for HDFStorerpzcomplib only supports z compression.rurr)rrfiltersZ all_complibsZdefault_complibr9_pathrrr_complibr_filtersopen)selfrrrrrrrprFrFrG__init__/s&  zHDFStore.__init__cCs|jSrIrrrFrFrG __fspath__QszHDFStore.__fspath__cCs||jdusJ|jjS)zreturn the root nodeN)_check_if_openrrootrrFrFrGrTsz HDFStore.rootcCs|jSrIrrrFrFrGfilename[szHDFStore.filenamer~cCs ||SrI)getrr~rFrFrG __getitem___szHDFStore.__getitem__cCs|||dSrIr)rr~rrFrFrG __setitem__bszHDFStore.__setitem__cCs ||SrI)removerrFrFrG __delitem__eszHDFStore.__delitem__rOc CsDz ||WSttfy"Yn0tdt|jd|ddS)z$allow attribute access to get stores'z' object has no attribute 'N)rrrerqtyperb)rrPrFrFrG __getattr__hs zHDFStore.__getattr__r~rcCs8||}|dur4|j}||ks0|dd|kr4dSdS)zx check for existence of this key can match the exact pathname or the pathnm w/o the leading '/' NrUTF)get_noder)rr~noderPrFrFrG __contains__rs  zHDFStore.__contains__rcCs t|SrI)r]rrrFrFrG__len__~szHDFStore.__len__cCst|j}t|d|dS)N File path:  )r;rr)rpstrrFrFrG__repr__s zHDFStore.__repr__cCs|SrIrFrrFrFrG __enter__szHDFStore.__enter__cCs |dSrI)r)rexc_type exc_value tracebackrFrFrG__exit__szHDFStore.__exit__pandas list[str])includercCs^|dkrdd|DS|dkrJ|jdus0Jdd|jjddd DStd |d dS) a# Return a list of keys corresponding to objects stored in HDFStore. Parameters ---------- include : str, default 'pandas' When kind equals 'pandas' return pandas objects. When kind equals 'native' return native HDF5 Table objects. .. versionadded:: 1.1.0 Returns ------- list List of ABSOLUTE path-names (e.g. have the leading '/'). Raises ------ raises ValueError if kind has an illegal value rcSsg|] }|jqSrFrrWnrFrFrGrZz!HDFStore.keys..ZnativeNcSsg|] }|jqSrFrrrFrFrGrZs/Table) classnamez8`include` should be either 'pandas' or 'native' but is 'r)rrZ walk_nodesr)rrrFrFrGkeyss z HDFStore.keyscCs t|SrI)iterrrrFrFrG__iter__szHDFStore.__iter__ccs|D]}|j|fVqdS)z' iterate on key->group N)rr)rgrFrFrGitemss zHDFStore.items)rcKst}|j|krR|jdvr$|dvr$n(|dvrL|jrLtd|jd|jd||_|jr`||jr|jdkrtj|j|j|j d|_ t r|jrd }t ||j |j|jfi||_d S) a9 Open the file in the specified mode Parameters ---------- mode : {'a', 'w', 'r', 'r+'}, default 'a' See HDFStore docstring or tables.open_file for info about modes **kwargs These parameters will be passed to the PyTables open_file method. )ruw)rr)rzRe-opening the file [z ] with mode [z] will delete the current file!r)rzGCannot open HDF5 file, which is already opened, even in read-only mode.N)rtrrr`rrrFiltersrrrrsr open_filer)rrrrpmsgrFrFrGrs*    z HDFStore.opencCs|jdur|jd|_dS)z0 Close the PyTables file handle N)rrrrFrFrGrs  zHDFStore.closecCs|jdurdSt|jjS)zF return a boolean indicating whether the file is open NF)rryZisopenrrFrFrGrs zHDFStore.is_open)fsynccCsT|jdurP|j|rPtt t|jWdn1sF0YdS)a Force all buffered modifications to be written to disk. Parameters ---------- fsync : bool (default False) call ``os.fsync()`` on the file handle to force writing to disk. Notes ----- Without ``fsync=True``, flushing may not guarantee that the OS writes to disk. With fsync, the operation will block until the OS claims the file has been written; however, other caching layers may still interfere. N)rflushrrrrfileno)rrrFrFrGrs    zHDFStore.flushcCsVt<||}|dur*td|d||WdS1sH0YdS)z Retrieve pandas object stored in file. Parameters ---------- key : str Returns ------- object Same type as object stored in file. NNo object named in the file)rrr _read_grouprr~rrFrFrGr s  z HDFStore.get)r~rc st||} | dur"td|dt|dd}|| fdd} t|| |j|||||d } | S) a Retrieve pandas object stored in file, optionally based on where criteria. .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- key : str Object being retrieved from file. where : list or None List of Term (or convertible) objects, optional. start : int or None Row number to start selection. stop : int, default None Row number to stop selection. columns : list or None A list of columns that if not None, will limit the return columns. iterator : bool or False Returns an iterator. chunksize : int or None Number or rows to include in iteration, return an iterator. auto_close : bool or False Should automatically close the store when finished. Returns ------- object Retrieved object from file. NrrrUrScsj|||dS)N)rrr^rread_start_stop_whererrErFrGfuncZszHDFStore.select..funcr^nrowsrrrrr)rrr__create_storer infer_axes TableIteratorr get_result) rr~r^rrrrrrrritrFrrGr"s(.    zHDFStore.selectr~rrcCs8t|dd}||}t|ts(td|j|||dS)a return the selection as an Index .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- key : str where : list of Term (or convertible) objects, optional start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection rUrSz&can only read_coordinates with a tabler^rr)r_ get_storerrArrread_coordinates)rr~r^rrtblrFrFrGselect_as_coordinatesms    zHDFStore.select_as_coordinates)r~columnrrcCs,||}t|tstd|j|||dS)a~ return a single column from the table. This is generally only useful to select an indexable .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- key : str column : str The column of interest. start : int or None, default None stop : int or None, default None Raises ------ raises KeyError if the column is not found (or key is not a valid store) raises ValueError if the column can not be extracted individually (it is part of a data block) z!can only read_column with a tablerrr)r rArr read_column)rr~rrrr rFrFrG select_columns#  zHDFStore.select_column)rc  szt|dd}t|ttfr.t|dkr.|d}t|trRj||||||| dSt|ttfshtdt|sxtd|dur|d}fdd |D |} d} t | |fgt |D]\\} } | durt d | d | jstd | jd | dur | j} q| j| krtdqdd D}tdd|Ddfdd}t| ||| ||||| d }|jddS)a Retrieve pandas objects from multiple tables. .. warning:: Pandas uses PyTables for reading and writing HDF5 files, which allows serializing object-dtype data with pickle when using the "fixed" format. Loading pickled data received from untrusted sources can be unsafe. See: https://docs.python.org/3/library/pickle.html for more. Parameters ---------- keys : a list of the tables selector : the table to apply the where criteria (defaults to keys[0] if not supplied) columns : the columns I want back start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection iterator : bool, return an iterator, default False chunksize : nrows to include in iteration, return an iterator auto_close : bool, default False Should automatically close the store when finished. Raises ------ raises KeyError if keys or selector is not found or keys is empty raises TypeError if keys is not a list or tuple raises ValueError if the tables are not ALL THE SAME DIMENSIONS rUrSr)r~r^rrrrrrzkeys must be a list/tuplez keys must have a non-zero lengthNcsg|]}|qSrF)r rWkrrFrGrZrz/HDFStore.select_as_multiple..zInvalid table []zobject [z>] is not a table, and cannot be used in all select as multiplez,all tables must have exactly the same nrows!cSsg|]}t|tr|qSrF)rArrWxrFrFrGrZrcSsh|]}|jddqSr)non_index_axesrWrlrFrFrG rz.HDFStore.select_as_multiple..cs*fddD}t|ddS)Ncsg|]}|jdqS)r^rrrrr)rrrrrFrGrZsz=HDFStore.select_as_multiple..func..F)axisverify_integrity)r- _consolidate)rrrobjs)rrtblsrrGrsz)HDFStore.select_as_multiple..funcrT coordinates)r_rAr[r\r]rNrrrr  itertoolschainzipris_tablepathnamerrr)rrr^selectorrrrrrrrErrlrZ_tblsrrrF)rrrr!rGselect_as_multiplesd+           zHDFStore.select_as_multipleTrnrvrzr|)r~rrrrr track_timesrcCsH|durtdpd}||}|j||||||||| | | | | |ddS)aO Store object in HDFStore. Parameters ---------- key : str value : {Series, DataFrame} format : 'fixed(f)|table(t)', default is 'fixed' Format to use when storing object in HDFStore. Value can be one of: ``'fixed'`` Fixed format. Fast writing/reading. Not-appendable, nor searchable. ``'table'`` Table format. Write as a PyTables Table structure which may perform worse but allow more flexible operations like searching / selecting subsets of the data. append : bool, default False This will force Table format, append the input data to the existing. data_columns : list of columns or True, default None List of columns to create as data columns, or True to use all columns. See `here `__. encoding : str, default None Provide an encoding for strings. track_times : bool, default True Parameter is propagated to 'create_table' method of 'PyTables'. If set to False it enables to have the same h5 files (same hashes) independent on creation time. .. versionadded:: 1.1.0 Nio.hdf.default_formatri) rrrrrrrrrLrr+r)r_validate_format_write_to_group)rr~rrrrrrrrrrLrr+rrFrFrGr/s&0  z HDFStore.putc Cst|dd}z||}Wnty.Yntty@Ynbty}zJ|dur`td|||}|dur|jddWYd}~dSWYd}~n d}~00t |||r|j jddn|j std|j |||dSdS) a: Remove pandas object partially by specifying the where condition Parameters ---------- key : str Node to remove or delete rows from where : list of Term (or convertible) objects, optional start : integer (defaults to None), row number to start selection stop : integer (defaults to None), row number to stop selection Returns ------- number of rows removed (or None if not a Table) Raises ------ raises KeyError if key is not a valid store rUrSNz5trying to remove a node with a non-None where clause!T recursivez7can only remove with where on objects written as tablesr ) r_r rAssertionError ExceptionrrZ _f_removecomall_nonerr'delete)rr~r^rrrEerrrrFrFrGrss2     $zHDFStore.remover{)r~rrrrrrcCsl| durtd|dur td}|dur4tdp2d}||}|j||||||||| | | | ||||ddS)a6 Append to Table in file. Node must already exist and be Table format. Parameters ---------- key : str value : {Series, DataFrame} format : 'table' is the default Format to use when storing object in HDFStore. Value can be one of: ``'table'`` Table format. Write as a PyTables Table structure which may perform worse but allow more flexible operations like searching / selecting subsets of the data. append : bool, default True Append the input data to the existing. data_columns : list of columns, or True, default None List of columns to create as indexed data columns for on-disk queries, or True to use all columns. By default only the axes of the object are indexed. See `here `__. min_itemsize : dict of columns that specify minimum str sizes nan_rep : str to use as str nan representation chunksize : size to chunk the writing expectedrows : expected TOTAL row size of this table encoding : default None, provide an encoding for str dropna : bool, default False Do not write an ALL nan row to the store settable by the option 'io.hdf.dropna_table'. Notes ----- Does *not* check if data being appended overlaps with existing data in the table, so be careful Nz>columns is not a supported keyword in append, try data_columnszio.hdf.dropna_tabler,rj)raxesrrrrrrr expectedrowsrrrLr)rrr-r.)rr~rrr7rrrrrrrrr8rrrLrrFrFrGrs68  zHDFStore.appenddict)dc s|durtdt|ts"td||vr2tdtttjttt d}d} g} | D]0\} dur| durtd| } qh| qh| durֈj |} | t| } t| | } | | || <|dur||}|r*fdd|D}t|}|D]}||}qj||d d}| D]h\} | |krT|nd}j|d }|durfd d | Dnd}|j| |f||d |q>dS)a Append to multiple tables Parameters ---------- d : a dict of table_name to table_columns, None is acceptable as the values of one node (this will get all the remaining columns) value : a pandas object selector : a string that designates the indexable table; all of its columns will be designed as data_columns, unless data_columns is passed, in which case these are used data_columns : list of columns to create as data columns, or True to use all columns dropna : if evaluates to True, drop rows from all tables if any single row in each table has all NaN. Default False. Notes ----- axes parameter is currently not accepted Nztaxes is currently not accepted as a parameter to append_to_multiple; you can create the tables independently insteadzQappend_to_multiple must have a dictionary specified as the way to split the valuez=append_to_multiple requires a selector that is in passed dictrzLrz.HDFStore.append_to_multiple..rrcsi|]\}}|vr||qSrFrFrWr~r)vrFrG \rz/HDFStore.append_to_multiple..)rr)rrAr9rr[setrangendim _AXES_MAPrrextendr7 differencer(sorted get_indexertakevaluesnext intersectionlocpopreindexr)rr:rr)rr7rrrZ remain_keyZ remain_valuesrorderedZorddZidxsZ valid_indexrrdcvalfilteredrF)rArrGappend_to_multiplesZ &    zHDFStore.append_to_multiplerx)r~optlevelkindcCsBt||}|durdSt|ts.td|j|||ddS)a Create a pytables index on the table. Parameters ---------- key : str columns : None, bool, or listlike[str] Indicate which columns to create an index on. * False : Do not create any indexes. * True : Create indexes on all columns. * None : Create indexes on all columns. * listlike : Create indexes on the given columns. optlevel : int or None, default None Optimization level, if None, pytables defaults to 6. kind : str or None, default None Kind of index, if None, pytables defaults to "medium". Raises ------ TypeError: raises if the node is not a table Nz1cannot create table index on a Fixed format store)rrWrX)rtr rArr create_index)rr~rrWrXrErFrFrGcreate_table_indexbs  zHDFStore.create_table_indexcCs<t||jdusJtdus(Jdd|jDS)z Return a list of all the top-level nodes. Each node returned is not a pandas storage object. Returns ------- list List of objects. NcSsPg|]H}t|tjjst|jddsHt|ddsHt|tjjr|jdkr|qS) pandas_typeNrj) rArolinkLinkgetattr_v_attrsrjrr)rWrrFrFrGrZs  z#HDFStore.groups..)rtrrro walk_groupsrrFrFrGrs  zHDFStore.groupsrccst||jdusJtdus(J|j|D]}t|jdddurLq4g}g}|jD]B}t|jdd}|durt |tj j r| |j q^| |j q^|jd||fVq4dS)aS Walk the pytables group hierarchy for pandas objects. This generator will yield the group path, subgroups and pandas object names for each group. Any non-pandas PyTables objects that are not a group will be ignored. The `where` group itself is listed first (preorder), then each of its child groups (following an alphanumerical order) is also traversed, following the same procedure. Parameters ---------- where : str, default "/" Group where to start walking. Yields ------ path : str Full path to a group (without trailing '/'). groups : list Names (strings) of the groups contained in `path`. leaves : list Names (strings) of the pandas objects contained in `path`. Nr[r)rtrrror`r^r_Z _v_childrenrLrArGrouprrrrstrip)rr^rrZleaveschildr[rFrFrGwalks  z HDFStore.walkz Node | NonecCs~||dsd|}|jdus(Jtdus4Jz|j|j|}Wntjjy`YdS0t|tj szJt ||S)z9return the node with the key or None if it does not existrN) r startswithrrorr exceptionsZNoSuchNodeErrorrAr>r)rr~rrFrFrGrs  zHDFStore.get_nodeGenericFixed | TablecCs8||}|dur"td|d||}||S)z| | vrj|rj| | || } t| trd} |rdd| j D} | j | | | t | dd| j dq>| j | | | j dq>| S) a; Copy the existing store to a new file, updating in place. Parameters ---------- propindexes : bool, default True Restore indexes in copied file. keys : list, optional List of keys to include in the copy (defaults to all). overwrite : bool, default True Whether to overwrite (remove and replace) existing nodes in the new store. mode, complib, complevel, fletcher32 same as in HDFStore.__init__ Returns ------- open file handle of the new store )rrrrNFcSsg|]}|jr|jqSrF) is_indexedrPrWrurFrFrGrZ rz!HDFStore.copy..r)rrrLrK)rr[rrAr\r rrrr7rr^rLr)rrrrrhrrrr overwriteZ new_storerrEdatarrFrFrGcopys6       z HDFStore.copyc Cs t|j}t|d|d}|jrt|}t|rg}g}|D]}z<||}|dur|t|j pj||t|p|dWqDt yYqDt y}z0||t|} |d| dWYd}~qDd}~00qD|t d||7}n|d7}n|d 7}|S) zg Print detailed information on the store. Returns ------- str rrNzinvalid_HDFStore nodez[invalid_HDFStore node: r EmptyzFile is CLOSED) r;rrrrIrr]r rr(r1r2r:) rroutputZlkeysrrLrrEZdetailZdstrrFrFrGinfo-s.     * z HDFStore.infocCs|jst|jddS)Nz file is not open!)rrerrrFrFrGrWszHDFStore._check_if_open)rrc CsLzt|}Wn6tyF}ztd|d|WYd}~n d}~00|S)zvalidate / deprecate formatsz#invalid HDFStore format specified [rN) _FORMAT_MAPlowerrr)rrr6rFrFrGr-[s (zHDFStore._validate_formatr@zDataFrame | Series | None)rrLrrc s"durtttfstdfdd}ttjdd}ttjdd}|durƈdurttdustJtddsttj j rd}d }qtd n$ttrd }nd }dkr|d 7}d|vr(t t d} z | |} Wn0t y} z|d| WYd} ~ n d} ~ 00| |||dS|durˆdur|dkrtdd} | dur| jdkrnd}n| jdkrd}nB|dkrtdd} | dur| jdkrd}n| jdkrd}ttttttd} z | |} Wn0t y} z|d| WYd} ~ n d} ~ 00| |||dS)z"return a suitable class to operateNz(value must be None, Series, or DataFramec s$td|ddtdS)Nz(cannot properly create the storer for: [z ] [group->,value->z ,format->)rr)rlrrrrFrGerrorssz&HDFStore._create_storer..errorr[ table_typerj frame_table generic_tablezKcannot create a storer if the object is not existing nor a value are passedseriesframeZ_table)rzr{ _STORER_MAPrLr series_tablerrUappendable_seriesappendable_multiseriesappendable_frameappendable_multiframe)ryrrrrworm _TABLE_MAP)rAr+r&rrHr^r_rtrorjr SeriesFixed FrameFixedrnlevels GenericTableAppendableSeriesTableAppendableMultiSeriesTableAppendableFrameTableAppendableMultiFrameTable WORMTable)rrrrrLrrvptttr|clsr6rrrFrurGresr                     zHDFStore._create_storerr})r~rrrrr+rcCst|ddr|dks|rdS|||}|j|||||d}|rr|jrZ|jrb|dkrb|jrbtd|jsz|n||js|rtd|j|||||| | | | | |||d t|t r|r|j |ddS) Nemptyrjr}rizCan only append to Tablesz0Compression not supported on Fixed format stores) objr7rrrrrrr8rrrr+)r) r^_identify_grouprr' is_existsrset_object_infowriterArrY)rr~rrr7rrrrrrrr8rrrrLrr+rrErFrFrGr.s:   zHDFStore._write_to_groupr>rcCs||}||SrI)rrr)rrrErFrFrGrs zHDFStore._read_group)r~rrcCsN||}|jdusJ|dur8|s8|jj|ddd}|durJ||}|S)z@Identify HDF5 group based on key, delete/create group if needed.NTr/)rr remove_node_create_nodes_and_group)rr~rrrFrFrGrs   zHDFStore._identify_groupcCsv|jdusJ|d}d}|D]P}t|s.q |}|dsD|d7}||7}||}|durl|j||}|}q |S)z,Create nodes from key and return group name.Nr)rsplitr]endswithrZ create_group)rr~pathsrpnew_pathrrFrFrGrs   z HDFStore._create_nodes_and_group)ruNNF)r)ru)F)NNNNFNF)NNN)NN)NNNNNFNF) NTFNNNNNNrnTF)NNN)NNTTNNNNNNNNNNrn)NNF)NNN)r)rTNNNFT)NNr@rn)NTFNNNNNNFNNNrnT)2rbrcrd__doc____annotations__rrpropertyrrrrrrrrrrrrrr iteritemsrrrrrrrrr*rrrrVrZrrdrr rmrqrr-rr.rrrrFrFrFrGrs A"    "-N$+~D=Zd( 0 =* a>rc@sbeZdZUdZded<ded<ded<dddd dd d d d ZddZddZdd dddZdS)raa Define the iteration interface on a table Parameters ---------- store : HDFStore s : the referred storer func : the function to execute the query where : the where of the query nrows : the rows to iterate on start : the passed start value (default is None) stop : the passed stop value (default is None) iterator : bool, default False Whether to use the default iterator. chunksize : the passed chunking value (default is 100000) auto_close : bool, default False Whether to automatically close the store at the end of iteration. rwrrrrgrENFry)rrErrrc Cs||_||_||_||_|jjrN|dur,d}|dur8d}|durD|}t||}||_||_||_d|_ |sr| dur| dur~d} t | |_ nd|_ | |_ dS)Nr順) rrErr^r'minrrrr#rRrr) rrrErr^rrrrrrrFrFrGrDs,    zTableIterator.__init__ccsv|j}|jdurtd||jkrjt||j|j}|dd|j||}|}|dust|sbq|Vq|dS)Nz*Cannot iterate until get_result is called.) rr#rrrrrr]r)rrrrrFrFrGrns  zTableIterator.__iter__cCs|jr|jdSrI)rrrrrFrFrGr~szTableIterator.closer"cCs|jdur4t|jtstd|jj|jd|_|S|rft|jtsLtd|jj|j|j|j d}n|j}| |j|j |}| |S)Nz0can only use an iterator or chunksize on a table)r^z$can only read_coordinates on a tabler ) rrArErrr r^r#rrrr)rr#r^resultsrFrFrGrs    zTableIterator.get_result)NNFNF)F) rbrcrdrrrrrrrFrFrFrGr,s  *rc @steZdZUdZdZdZgdZded<ded<dHddd d d Ze d d ddZ e dd ddZ d dddZ dd ddZ dddddZdd ddZe dd ddZd ddd!d"d#Zd$d%Ze d&d'Ze d(d)Ze d*d+Ze d,d-Zd.d/ZdId0d1Zd2d3Zd4dd5d6d7ZdJd8d9Zdd:d;d<Zd=d>Zd?d@ZdAdBZd4dCdDdEZ d4dCdFdGZ!dS)KIndexCola an index column description class Parameters ---------- axis : axis which I reference values : the ndarray like converted values kind : a string description of this type typ : the pytables type pos : the position in the pytables T)freqtz index_namerNrPcnameNrx)rPrcCst|tstd||_||_||_||_|p0||_||_||_ ||_ | |_ | |_ | |_ | |_| |_||_|dur|||t|jtsJt|jtsJdS)Nz`name` must be a str.)rArNrrLrXtyprPrrposrrrrRrjrmetadataset_pos)rrPrLrXrrrrrrrrRrjrrrFrFrGrs(   zIndexCol.__init__rRrcCs|jjSrI)ritemsizerrFrFrGrszIndexCol.itemsizecCs |jdS)N_kindrOrrFrFrG kind_attrszIndexCol.kind_attr)rcCs$||_|dur |jdur ||j_dS)z,set the position of this column in the TableN)rrZ_v_pos)rrrFrFrGrszIndexCol.set_poscCs@ttt|j|j|j|j|jf}dddt gd|DS)N,cSsg|]\}}|d|qSz->rFr@rFrFrGrZsz%IndexCol.__repr__..)rPrrrrX) r\mapr;rPrrrrXjoinr&rtemprFrFrGrs zIndexCol.__repr__rryotherrcstfdddDS)compare 2 col itemsc3s&|]}t|dt|dkVqdSrIr^rjrrrFrGr>sz"IndexCol.__eq__..)rPrrrr;rrrFrrG__eq__szIndexCol.__eq__cCs || SrI)rrrFrFrG__ne__szIndexCol.__ne__cCs"t|jdsdSt|jj|jjS)z%return whether I am an indexed columnr=F)hasattrrjr^r=rrirrFrFrGris zIndexCol.is_indexed np.ndarrayrLrLrc Cst|tjsJt||jjdur.||j}t|j}t ||||}i}t|j |d<|j durpt|j |d<t }t |jst|jrt}n|jdkrd|vrdd}z||fi|}Wn2tyd|vrd|d<||fi|}Yn0t||j} | | fS)zV Convert the data from this selection to the appropriate pandas type. NrPri8c[stfd|i|S)NZordinal)r*)rkwdsrFrFrGr$sz"IndexCol.convert..)rArBndarrayrdtypefieldsrrHrX_maybe_convertrrr(rrr'r_set_tzr) rrLrrLrval_kindrfactoryZ new_pd_indexZfinal_pd_indexrFrFrGconvert s,      zIndexCol.convertcCs|jS)zreturn the valuesrLrrFrFrG take_data4szIndexCol.take_datacCs|jjSrI)rjr_rrFrFrGattrs8szIndexCol.attrscCs|jjSrIrj descriptionrrFrFrGr<szIndexCol.descriptioncCst|j|jdS)z!return my current col descriptionN)r^rrrrFrFrGcol@sz IndexCol.colcCs|jSzreturn my cython valuesrrrFrFrGcvaluesEszIndexCol.cvaluescCs t|jSrI)rrLrrFrFrGrJszIndexCol.__iter__cCsPt|jdkrLt|tr$||j}|durL|jj|krLtj ||j d|_dS)z maybe set a string col itemsize: min_itemsize can be an integer or a dict with this columns name with an integer size stringN)rr) rHrXrAr9rrPrrrt StringColr)rrrFrFrGmaybe_set_sizeMs   zIndexCol.maybe_set_sizecCsdSrIrFrrFrFrGvalidate_namesZszIndexCol.validate_namesAppendableTable)handlerrcCs:|j|_||||||||dSrI)rj validate_col validate_attrvalidate_metadatawrite_metadataset_attr)rrrrFrFrGvalidate_and_set]s    zIndexCol.validate_and_setcCs^t|jdkrZ|j}|durZ|dur*|j}|j|krTtd|d|jd|jd|jSdS)z:validate this column: return the compared against itemsizerNz#Trying to store a string with len [z] in [z)] column but this column has a limit of [zC]! Consider using min_itemsize to preset the sizes on these columns)rHrXrrrr)rrcrFrFrGres zIndexCol.validate_colrcCsB|r>t|j|jd}|dur>||jkr>td|d|jddS)Nzincompatible kind in col [ - r)r^rrrXr)rrZ existing_kindrFrFrGrxs zIndexCol.validate_attrc Cs|jD]}t||d}||ji}||}||vr|dur||kr|dvrt|||f}tj|tt dd||<t ||dqt d|jd|d|d|d q|dus|dur|||<qdS) z set/update the info for this indexable with the key/value if there is a conflict raise/warn as needed N)rr stacklevelzinvalid info for [z] for [z], existing_value [z] conflicts with new value [r) _info_fieldsr^ setdefaultrPrattribute_conflict_docwarningswarnrgrsetattrr)rrqr~ridxZexisting_valuewsrFrFrG update_infos*   zIndexCol.update_infocCs$||j}|dur |j|dS)z!set my state from the passed infoN)rrP__dict__update)rrqrrFrFrGset_infos zIndexCol.set_infocCst|j|j|jdS)zset the kind for this columnN)rrrrXrrFrFrGrszIndexCol.set_attr)rcCsB|jdkr>|j}||j}|dur>|dur>t||s>tddS)z:validate that kind=category does not change the categoriescategoryNzEcannot append a categorical with different categories to the existing)rr read_metadatarr%r)rrZ new_metadataZ cur_metadatarFrFrGrs  zIndexCol.validate_metadatacCs|jdur||j|jdS)zset the meta dataN)rrr)rrrFrFrGrs zIndexCol.write_metadata) NNNNNNNNNNNNN)N)N)"rbrcrdris_an_indexableis_data_indexablerrrrrrrrrrrirrrrrrrrrrrrrrrrrrFrFrFrGrsf  , +       !rc@s<eZdZdZeddddZddddd d Zd d Zd S)GenericIndexColz:an index which is not represented in the data of the tableryrcCsdSNFrFrrFrFrGriszGenericIndexCol.is_indexedrrNrcCs2t|tjsJt|ttt|}||fS)z Convert the data from this selection to the appropriate pandas type. Parameters ---------- values : np.ndarray nan_rep : str encoding : str errors : str )rArBrrr/aranger])rrLrrLrrFrFrGrs zGenericIndexCol.convertcCsdSrIrFrrFrFrGrszGenericIndexCol.set_attrN)rbrcrdrrrirrrFrFrFrGrs rc s,eZdZdZdZdZddgZd9dddfd d Zedd d d Z edd ddZ dd ddZ dddddZ ddddZ ddZeddddd Zed!d"Zedd#d$d%d&Zeddd$d'd(Zed)d*Zed+d,Zed-d.Zed/d0Zd1d2Zd3ddd4d5d6Zd7d8ZZS):DataCola3 a data holding column, by definition this is not indexable Parameters ---------- data : the actual data cname : the column name in the table to hold the data (typically values) meta : a string description of the metadata metadata : the actual metadata FrrRNrNzDtypeArg | None)rPrc s2tj||||||||| | | d | |_| |_dS)N) rPrLrXrrrrrRrjrr)superrrrl)rrPrLrXrrrrrRrjrrrrl __class__rFrGrs zDataCol.__init__rcCs |jdS)N_dtyperOrrFrFrG dtype_attr szDataCol.dtype_attrcCs |jdS)N_metarOrrFrFrG meta_attr szDataCol.meta_attrcCs@ttt|j|j|j|j|jf}dddt gd|DS)NrcSsg|]\}}|d|qSrrFr@rFrFrGrZ sz$DataCol.__repr__..)rPrrrXshape) r\rr;rPrrrXrrr&rrFrFrGr s zDataCol.__repr__rryrcstfdddDS)rc3s&|]}t|dt|dkVqdSrIrrjrrFrGr>& sz!DataCol.__eq__..)rPrrrrrrFrrGr$ szDataCol.__eq__rrlcCs@|dus J|jdusJt|\}}||_||_t||_dSrI)r_get_data_and_dtype_namerl_dtype_to_kindrX)rrl dtype_namerFrFrGset_data+ s   zDataCol.set_datacCs|jS)zreturn the datarrrFrFrGr5 szDataCol.take_datar<)rLrcCs|j}|j}|j}|jdkr&d|jf}t|trJ|j}|j||jj d}ntt |sZt |rf| |}nXt |rz||}nDt|rtj||dd}n&t|r|||}n|j||j d}|S)zW Get an appropriately typed and shaped pytables.Col object for values. rUrXrrr)rrrrEsizerAr0codes get_atom_datarPrrget_atom_datetime64r#get_atom_timedelta64rrtZ ComplexColr"get_atom_string)rrLrrrr atomrFrFrG _get_atom9 s$     zDataCol._get_atomcCstj||ddS)NrrrtrrrrrFrFrGrY szDataCol.get_atom_stringz type[Col]rXrcCsR|dr$|dd}d|d}n"|dr4d}n|}|d}tt|S)z0return the PyTables column class for this columnuintNZUIntr<periodInt64Col)re capitalizer^rt)rrXZk4Zcol_nameZkcaprFrFrGget_atom_coltype] s    zDataCol.get_atom_coltypecCs|j|d|ddS)NrrrrrrrXrFrFrGr l szDataCol.get_atom_datacCstj|ddSNrrrtrrrrFrFrGr p szDataCol.get_atom_datetime64cCstj|ddSrrrrFrFrGr t szDataCol.get_atom_timedelta64cCst|jddS)Nr)r^rlrrFrFrGrx sz DataCol.shapecCs|jSrrrrFrFrGr| szDataCol.cvaluescCs`|r\t|j|jd}|dur2|t|jkr2tdt|j|jd}|dur\||jkr\tddS)zAvalidate that we have the same order as the existing & same dtypeNz4appended items do not match existing items in table!z@appended items dtype do not match existing items dtype in table!)r^rrr[rLrrr)rrZexisting_fieldsZexisting_dtyperFrFrGr szDataCol.validate_attrrrcCst|tjsJt||jjdur.||j}|jdus.cSsg|]}t|qSrFr fromtimestampr(rFrFrGrZ rr) categoriesrRFrmOrrrLr)"rArBrrrrrrrrrXrHrrrRrrasarrayobjectrravelr(Zfloat64r.anyastyperRZcumsum_valuesr0Z from_codesr_unconvert_string_arrayrL)rrLrrLr convertedrrXrrrRrrr,r maskrFrFrGr sf              zDataCol.convertcCsHt|j|j|jt|j|j|j|jdus2Jt|j|j|jdS)zset the data for this columnN)rrrrLrrrrrrFrFrGr szDataCol.set_attr) NNNNNNNNNNNN)rbrcrdrrrrrrrrrrrr classmethodrrrr r r rrrrr __classcell__rFrFrrGrsX          erc@sTeZdZdZdZddZeddZeddd d d Zed d Z eddZ dS)DataIndexableColz+represent a data column that can be indexedTcCst|jstddS)N-cannot have non-object label DataIndexableCol)r(rLZ is_objectrrrFrFrGr szDataIndexableCol.validate_namescCstj|dS)N)rrrrFrFrGr sz DataIndexableCol.get_atom_stringrNr<rcCs|j|dS)NrrrrFrFrGr  szDataIndexableCol.get_atom_datacCs tSrIrrrFrFrGr  sz$DataIndexableCol.get_atom_datetime64cCs tSrIrrrFrFrGr  sz%DataIndexableCol.get_atom_timedelta64N) rbrcrdrrrr9rr r r rFrFrFrGr; s  r;c@seZdZdZdS)GenericDataIndexableColz(represent a generic pytables data columnN)rbrcrdrrFrFrFrGr= sr=c@seZdZUdZded<dZded<ded<ded <ded <d ed <d ed<ded<dZdLd d dddddZeddddZ eddddZ eddZ dddd Z d!d"Z d#d$Zed%d&Zed'd(Zed)d*Zed+d,Zeddd-d.Zeddd/d0Zed1d2Zd3d4Zd5d6Zed7d8Zeddd9d:Zed;d<Zd=d>ZdMd@dAZdBdCZdNdDdDdEdFdGZdHdIZdOdDdDdEdJdKZ d?S)PFixedz represent an object in my store facilitate read/write of various types of objects this is an abstract base class Parameters ---------- parent : HDFStore group : Node The group node where the table resides. rN pandas_kindri format_typetype[DataFrame | Series]obj_typerRrErLrrr>rrFr@rn)rrrLrcCsZt|tsJt|tdus"Jt|tjs:Jt|||_||_t||_||_ dSrI) rArrror>rrrMrLr)rrrrLrrFrFrGr3 s  zFixed.__init__ryrcCs*|jddko(|jddko(|jddkS)NrrU )versionrrFrFrGis_old_versionB szFixed.is_old_versionztuple[int, int, int]cCs`tt|jjdd}z0tdd|dD}t|dkrB|d}WntyZd}Yn0|S) zcompute and set our versionpandas_versionNcss|]}t|VqdSrIrRrrFrFrGr>K rz Fixed.version...rDr)rrr)rHr^rr_r\rr]rq)rrErFrFrGrEF s    z Fixed.versioncCstt|jjddS)Nr[)rHr^rr_rrFrFrGr[R szFixed.pandas_typecCs^||j}|durXt|ttfrDddd|D}d|d}|jdd|d S|jS) (return a pretty representation of myselfNrcSsg|] }t|qSrFr;rrFrFrGrZ\ rz"Fixed.__repr__..[r12.12z (shape->))rrrAr[r\rr[)rrEZjshaperFrFrGrV s zFixed.__repr__cCst|j|j_tt|j_dS)zset my pandas type & versionN)rNr?rr[_versionrGrrFrFrGra szFixed.set_object_infocCst|}|SrIr-)rZnew_selfrFrFrGrmf s z Fixed.copycCs|jSrI)rrrFrFrGrj sz Fixed.shapecCs|jjSrIrrrrFrFrGr(n szFixed.pathnamecCs|jjSrI)rrrrFrFrGrr sz Fixed._handlecCs|jjSrI)rrrrFrFrGrv szFixed._filterscCs|jjSrI)rrrrFrFrGrz szFixed._complevelcCs|jjSrI)rrrrFrFrGr~ szFixed._fletcher32cCs|jjSrI)rr_rrFrFrGr sz Fixed.attrscCsdSzset our object attributesNrFrrFrFrG set_attrs szFixed.set_attrscCsdS)zget our object attributesNrFrrFrFrG get_attrs szFixed.get_attrscCs|jS)zreturn my storablerrrFrFrGstorable szFixed.storablecCsdSrrFrrFrFrGr szFixed.is_existscCst|jddS)Nr)r^rTrrFrFrGr sz Fixed.nrowscCs|dur dSdS)z%validate against an existing storableNTrFrrFrFrGvalidate szFixed.validateNcCsdS)+are we trying to operate on an old version?TrF)rr^rFrFrGvalidate_version szFixed.validate_versioncCs|j}|durdS|dS)zr infer the axes of my storer return a boolean indicating if we have a valid storer or not NFT)rTrS)rrErFrFrGr s zFixed.infer_axesrwrrcCs tddS)Nz>cannot read on an abstract storer: subclasses should implementrrr^rrrrFrFrGr sz Fixed.readcKs tddS)Nz?cannot write on an abstract storer: subclasses should implementrYrrrFrFrGr sz Fixed.writecCs0t|||r$|jj|jdddStddS)zs support fully deleting the node in its entirety (only) - where specification must be None Tr/Nz#cannot delete on an abstract storer)r3r4rrrr)rr^rrrFrFrGr5 sz Fixed.delete)r@rn)N)NNNN)NNN)!rbrcrdrrr@r'rrrFrEr[rrrmrr(rrrrrrRrSrTrrrUrWrrrr5rFrFrFrGr> sl                r>c@s(eZdZUdZedediZddeDZgZ de d<dd d d Z d d Z ddZ ddZedd ddZddZddZddZd:ddddddZd;dddd d!d"d#Zdd d$d%d&Zdd'd$d(d)Zddd4d5d6d7d8d9ZdS)? GenericFixedza generified fixed versiondatetimercCsi|]\}}||qSrFrF)rWrrArFrFrGrB rzGenericFixed.r attributesrNrcCs|j|dS)N)_index_type_mapr)rrrFrFrG_class_to_alias szGenericFixed._class_to_aliascCst|tr|S|j|tSrI)rAr_reverse_index_maprr()raliasrFrFrG_alias_to_class s zGenericFixed._alias_to_classcCs|tt|dd}|tkr.d dd}|}n|tkrFd dd}|}n|}i}d|vrn|d|d<|turnt}d|vrt|dtr|d d|d<n |d|d<|tusJ||fS) N index_classr_cSs:tj|j|d}tj|dd}|dur6|d|}|S)NrrOUTC)r1 _simple_newrLr' tz_localize tz_convert)rLrrZdtaresultrFrFrGrk s z*GenericFixed._get_index_factory..fcSstj||d}tj|ddS)NrfrO)r2rhr*)rLrrZparrrFrFrGrk srrzutf-8)NN)NN) rdrHr^r'r*r(r,rAbytesrD)rrrerkrrrFrFrG_get_index_factory s*     zGenericFixed._get_index_factorycCs$|durtd|dur tddS)zE raise if any keywords are passed which are not-None Nzqcannot pass a column specification when reading a Fixed format store. this store must be selected in its entiretyzucannot pass a where specification when reading from a Fixed format store. this store must be selected in its entirety)r)rrr^rFrFrG validate_read szGenericFixed.validate_readrycCsdS)NTrFrrFrFrGr szGenericFixed.is_existscCs|j|j_|j|j_dSrQ)rLrrrrFrFrGrR s zGenericFixed.set_attrsc CsRtt|jdd|_tt|jdd|_|jD]}t||tt|j|dq.dS)retrieve our attributesrLNrrn)rMr^rrLrHrr^r)rrrFrFrGrS# s zGenericFixed.get_attrscKs |dSrI)rRrrrrFrFrGr* szGenericFixed.writeNrwr c Csddl}t|j|}|j}t|dd}t||jrD|d||}nztt|dd} t|dd} | durxtj| | d}n |||}| dkrt|d d} t || d d }n| d krtj |d d}|r|j S|SdS)z2read an array for the specified node (off of grouprN transposedF value_typerr%r rTr!r#r$) rpr^rr_rAZVLArrayrHrBrrr0T) rr~rrrprrrqretrrrrFrFrG read_array- s&      zGenericFixed.read_arrayr()r~rrrcCshtt|j|d}|dkr.|j|||dS|dkrVt|j|}|j|||d}|Std|dS)N_varietymultirXregularzunrecognized index variety: )rHr^rread_multi_indexrread_index_noder)rr~rrZvarietyrrrFrFrG read_indexO s zGenericFixed.read_index)r~rcCst|tr,t|j|dd|||nt|j|ddtd||j|j}|||j t |j |}|j |j _ |j|j _t|ttfr|t||j _t|tttfr|j|j _t|tr|jdurt|j|j _dS)Nrvrwrxr)rAr)rrwrite_multi_index_convert_indexrLr write_arrayrLr^rrXr_rPr'r*rarrer,rr_get_tz)rr~rr7rrFrFrG write_index] s     zGenericFixed.write_indexr)c Cst|j|d|jtt|j|j|jD]\}\}}}t|rJt d|d|}t |||j |j }| ||jt|j|} |j| j_|| j_t| j|d|||d|} | | |q,dS)N_nlevelsz=Saving a MultiIndex with an extension dtype is not supported._level_name_label)rrr enumerater&levelsr namesr rr}rLrr~rLr^rrXr_rP) rr~rilev level_codesrP level_keyZ conv_levelr label_keyrFrFrGr|t s"  zGenericFixed.write_multi_indexcCst|j|d}g}g}g}t|D]l}|d|} t|j| } |j| ||d} || || j|d|} |j| ||d} || q&t|||ddS)NrrrXrT)rr rr) r^rrDrrzrrPrur))rr~rrrrr rrrrrrrrFrFrGry s      zGenericFixed.read_multi_indexr>)rrrrc Cs|||}d|jvr>t|jjdkr>tj|jj|jjd}t|jj}d}d|jvrlt|jj }t|}|j}| |\}} |dkr|t |||j |j dfdti| } n |t |||j |j dfi| } || _ | S)Nrrr%rPrr}r)r_rBprodrrrrrHrXrQrPrm_unconvert_indexrLrr1) rrrrrlrXrPrrrrrFrFrGrz s:      zGenericFixed.read_index_noder)r~rcCsJtd|j}|j|j||t|j|}t|j|j _ |j |j _ dS)zwrite a 0-len arrayrUN) rBrrEr create_arrayrr^rNrr_rrr)rr~rZarrrrFrFrGwrite_array_empty s  zGenericFixed.write_array_emptyrvz Index | Noner})r~rrrc CsJt|dd}||jvr&|j|j||jdk}d}t|jrFtd|s^t|dr^|j }d}d}|j durt t  t j|j}Wdn1s0Y|dur|s|jj|j|||j|j d}||dd<n |||nL|jjtjkr`tj|dd} |rn,| d krn t| ||f} tj| ttd |j|j|t } | |nt |jr|j!|j||"d d t#|j|j$_%nt&|jr|j!|j||j't#|j|} t(|j)| j$_)d | j$_%n\t*|jr|j!|j||"d d t#|j|j$_%n&|r$|||n|j!|j|||t#|j|j$_+dS)NT)Z extract_numpyrFz]Cannot store a category dtype in a HDF5 dataset that uses format="fixed". Use format="table".rs)rZskipnarrrr r#),r5rrrr rrrrrsrrrrtZAtomZ from_dtypeZ create_carrayrrrrBZobject_r infer_dtypeperformance_docrrrrZcreate_vlarray ObjectAtomrrrviewr^r_rrrasi8rrr#rq) rr~rrrZ empty_arrayrqrca inferred_typerZvlarrrrFrFrGr~ sf       .        zGenericFixed.write_array)NN)NN)NN)NN)N)rbrcrdrr'r*r`rrbr^rrardrmrnrrrRrSrrur{rr|ryrzrr~rFrFrFrGr\ s2   .#& r\csNeZdZUdZdgZded<eddZd dddd d Zfd d Z Z S)rrzrPr c Cs.zt|jjfWSttfy(YdS0dSrI)r]rrLrrqrrFrFrGr4 szSeriesFixed.shapeNrwrXcCs<||||jd||d}|jd||d}t|||jdS)NrrXrL)rrP)rnr{rur+rP)rr^rrrrrLrFrFrGr; s zSeriesFixed.readc s<tj|fi||d|j|d||j|j_dS)NrrL)rrrrr~rPrrprrFrGrG s zSeriesFixed.write)NNNN) rbrcrdr?r^rrrrrr:rFrFrrGr. s   rcsReZdZUddgZded<eddddZdd d d d d Zfd dZZ S)BlockManagerFixedrEnblocksrRz Shape | NonercCsz|j}d}t|jD]8}t|jd|d}t|dd}|dur||d7}q|jj}t|dd}|durt|d|d}ng}|||WStyYdS0dS)Nrblock_itemsrrU) rErDrr^rZ block0_valuesr[rrq)rrErrrrrFrFrGrS s"    zBlockManagerFixed.shapeNrwrXcCs||||d}g}t|jD]<}||kr<||fnd\}} |jd||| d} || q(|d} g} t|jD]Z}|d|d} |jd|d|| d}| | | }t |j ||dd }| |q|t | dkrt | dd }|j| d d }|St |d|dd S) Nr)NNrrXrrr5rUrrr?F)rrm)rnrBZ_get_block_manager_axisrDrEr{rrrurJr&rsr]r-rQ)rr^rrrZ select_axisr7rrraxrdfs blk_itemsrLdfoutrFrFrGrn s(    zBlockManagerFixed.readc stj|fi|t|jtr*|d}|j}|s@|}|j|j _t |j D]0\}}|dkrr|j srt d|d||qTt|j|j _t |jD]D\}}|j|j}|jd|d|j|d|d|d|qdS)Nrrz/Columns index has to be unique for fixed formatrr5)rr)rrrA_mgrr7 _as_managerZis_consolidatedZ consolidaterErrr7Z is_uniquerrr]blocksrrrKmgr_locsr~rL)rrrrlrrblkrrrFrGr s    zBlockManagerFixed.write)NNNN) rbrcrdr^rrrrrr:rFrFrrGrN s %rc@seZdZdZeZdS)rr{N)rbrcrdr?r&rBrFrFrFrGr srcseZdZUdZdZdZded<ded<dZded <d Zd ed <d ed<ded<ded<ded<ded<dddddfdd Z e ddddZ ddd d!Z dd"d#d$Z d%d&Ze d'dd(d)Zd*d+d,d-d.Ze d/dd0d1Ze d'dd2d3Ze d4d5Ze d6d7Ze d8d9Ze d:d;Ze dd?Ze d'dd@dAZe dBdCZdDddEdFZdGdHZdIddJdKZdddLdMdNZddOdPdQdRZddSdTdUZ dVdWZ!dXdYZ"ddZd[Z#d\d]Z$e%d^d_Z&dd`dadbdcZ'ddddddedfdgdhZ(e)d'didjdkZ*dldmZ+ddnd'dodpdqZ,e-dnd'drdsdtZ.ddudvdwdxZ/ddd'dddDdydzd{Z0dddddd|d}d~Z1dddddddddZ2Z3S)raa represent a table: facilitate read/write of various types of tables Attrs in Table Node ------------------- These are attributes that are store in the main table node, they are necessary to recreate these tables when read back in. index_axes : a list of tuples of the (original indexing axis and index column) non_index_axes: a list of tuples of the (original index axis and columns on a non-indexing axis) values_axes : a list of the columns which comprise the data of this table data_columns : a list of the columns that we are allowing indexing (these become single columns in values_axes) nan_rep : the string to use for nan representations for string objects levels : the names of levels metadata : the names of the metadata columns Z wide_tablerjrNr@rwrUzint | list[Hashable]rTzlist[IndexCol] index_axeszlist[tuple[int, Any]]rz list[DataCol] values_axesr[rrr9rqNrnrr>)rrrc sPtj||||d|pg|_|p$g|_|p.g|_|p8g|_| pBi|_| |_dS)Nr})rrrrrrrqr) rrrrLrrrrrrqrrrFrGr s      zTable.__init__rcCs|jddS)N_r)rwrrrFrFrGtable_type_short szTable.table_type_shortc Cs|t|jrd|jnd}d|d}d}|jrZddd|jD}d|d}dd d|jD}|jd |d |jd |j d |j d|d|d S)rJrr_z,dc->[rrIcSsg|] }t|qSrFrNrrFrFrGrZ rz"Table.__repr__..rLcSsg|] }|jqSrFrOrjrFrFrGrZ rrMz (typ->z,nrows->z,ncols->z ,indexers->[rN) rr]rrrFrErr[rrncols)rZjdcrSverZjverZ jindex_axesrFrFrGr s(  zTable.__repr__)rcCs"|jD]}||jkr|SqdS)zreturn the axis for cN)r7rP)rrrurFrFrGr s   zTable.__getitem__c Cs|dur dS|j|jkr2td|jd|jddD]~}t||d}t||d}||kr6t|D]4\}}||}||krbtd|d|d|dqbtd|d|d|dq6dS) z"validate against an existing tableNz'incompatible table_type with existing [rr)rrrzinvalid combination of [z] on appending data [z] vs current table [)rwrr^rrr2)rrrsvovrsaxZoaxrFrFrGrU s:   zTable.validaterycCs t|jtS)z@the levels attribute is 1 or a list in the case of a multi-index)rArr[rrFrFrGis_multi_index+ szTable.is_multi_indexrvz tuple[DataFrame, list[Hashable]])rrc Cs`t|jj}z |}Wn.tyH}ztd|WYd}~n d}~00t|tsXJ||fS)ze validate that we can store the multi-index; reset and return the new object zBduplicate names/columns in the multi-index when storing as a tableN)r3Zfill_missing_namesrrZ reset_indexrrAr&)rrrZ reset_objr6rFrFrGvalidate_multiindex0 s zTable.validate_multiindexrRcCstdd|jDS)z-based on our axes, compute the expected nrowscSsg|]}|jjdqSr)rrrWrrFrFrGrZD rz(Table.nrows_expected..)rBrrrrFrFrGnrows_expectedA szTable.nrows_expectedcCs d|jvS)zhas this table been createdrjrrrFrFrGrF szTable.is_existscCst|jddSNrjr^rrrFrFrGrTK szTable.storablecCs|jS)z,return the table group (this is my storable))rTrrFrFrGrjO sz Table.tablecCs|jjSrI)rjrrrFrFrGrT sz Table.dtypecCs|jjSrIrrrFrFrGrX szTable.descriptioncCst|j|jSrI)r$r%rrrrFrFrGr7\ sz Table.axescCstdd|jDS)z.the number of total columns in the values axescss|]}t|jVqdSrI)r]rLrjrFrFrGr>c rzTable.ncols..)sumrrrFrFrGr` sz Table.ncolscCsdSrrFrrFrFrG is_transposede szTable.is_transposedcCs(ttdd|jDdd|jDS)z@return a tuple of my permutated axes, non_indexable at the frontcSsg|]}t|dqSrrHrjrFrFrGrZn rz*Table.data_orientation..cSsg|]}t|jqSrF)rRrrjrFrFrGrZo r)r\r$r%rrrrFrFrGdata_orientationi s zTable.data_orientationzdict[str, Any]csRdddddjD}fddjD}fddjD}t|||S)z.csg|]\}}|dfqSrIrF)rWrrL) axis_namesrFrGrZz rcs&g|]}|jtjvr|j|fqSrF)rPrCrrr(rrFrGrZ{ s)rrrr9)rZd1Zd2Zd3rF)rrrG queryabless s  zTable.queryablescCsdd|jDS)zreturn a list of my index colscSsg|]}|j|jfqSrF)rrrrFrFrGrZ rz$Table.index_cols..rrrFrFrG index_cols szTable.index_colsrcCsdd|jDS)zreturn a list of my values colscSsg|] }|jqSrFrrrFrFrGrZ rz%Table.values_cols..)rrrFrFrG values_cols szTable.values_colsrcCs|jj}|d|dS)z)return the metadata pathname for this keyz/meta/z/metarPrrFrFrG_get_metadata_path szTable._get_metadata_pathr)r~rLcCs,|jj||t|d|j|j|jddS)z Write out a metadata array to the key as a fixed-format Series. Parameters ---------- key : str values : ndarray rj)rrLrrN)rrrr+rLrr)rr~rLrFrFrGr s zTable.write_metadatarcCs0tt|jdd|ddur,|j||SdS)z'return the meta data array for this keyrN)r^rrrrrrFrFrGr szTable.read_metadatacCspt|j|j_||j_||j_|j|j_|j|j_|j|j_|j|j_|j |j_ |j |j_ |j |j_ dS)zset our table type & indexablesN) rNrwrrrrrrrLrrrqrrFrFrGrR s        zTable.set_attrscCst|jddpg|_t|jddp$g|_t|jddp8i|_t|jdd|_tt|jdd|_tt|jdd|_ t|jd dpg|_ d d |j D|_ d d |j D|_ dS) rorNrrqrrLrrnrcSsg|]}|jr|qSrFrrjrFrFrGrZ rz#Table.get_attrs..cSsg|]}|js|qSrFrrjrFrFrGrZ r)r^rrrrqrrMrLrHrr indexablesrrrrFrFrGrS szTable.get_attrscCs\|durX|jddkrX|jddkrX|jddkrXtddd|jD}t|tdS) rVNrrUrCrDrIcSsg|] }t|qSrFrrrFrFrGrZ rz*Table.validate_version..)rEincompatibility_docrrrrf)rr^rrFrFrGrW s*zTable.validate_versioncCsR|dur dSt|tsdS|}|D]&}|dkr4q&||vr&td|dq&dS)z validate the min_itemsize doesn't contain items that are not in the axes this needs data_columns to be defined NrLzmin_itemsize has the key [z%] which is not an axis or data_column)rAr9rr)rrqrrFrFrGvalidate_min_itemsize s  zTable.validate_min_itemsizec sg}jjjtjjD]j\}\}}t|}|}|durJdnd}|d}t|d} t|||| |j||d} || qt j t |fdd| fddtjj D|S) z/create/cache the indexables if they don't existNrr)rPrrrXrrjrrc st|tsJt}|vrt}t|}t|j}t|dd}t|dd}t|}|}t|dd} |||||||j | ||d } | S)Nrrr) rPrrLrXrrrjrrr) rArNrr;r^_maybe_adjust_namerErrrj) rrklassradj_namerLrrXmdrr)base_posrSdescr table_attrsrFrGrks0    zTable.indexables..fcsg|]\}}||qSrFrF)rWrr)rkrFrGrZ'rz$Table.indexables..)rrjrrrr^rrrrCrr]rGr) r _indexablesrrrPrrrrrX index_colrF)rrSrrkrrrGr s2      % zTable.indexablesrxrc CsV|s dS|durdS|dus(|dur8dd|jD}t|ttfsL|g}i}|dur`||d<|durp||d<|j}|D]}t|j|d}|dur"|jr|j }|j } |j } |dur| |kr| n| |d<|dur| |kr| n| |d<|jsP|j drtd |jfi|qz||jd d vrztd |d |d|dqzdS)aZ Create a pytables index on the specified columns. Parameters ---------- columns : None, bool, or listlike[str] Indicate which columns to create an index on. * False : Do not create any indexes. * True : Create indexes on all columns. * None : Create indexes on all columns. * listlike : Create indexes on the given columns. optlevel : int or None, default None Optimization level, if None, pytables defaults to 6. kind : str or None, default None Kind of index, if None, pytables defaults to "medium". Raises ------ TypeError if trying to create an index on a complex-type column. Notes ----- Cannot index Time64Col or ComplexCol. Pytables must be >= 3.0. NFTcSsg|]}|jr|jqSrF)rrrjrFrFrGrZNrz&Table.create_index..rWrXcomplexzColumns containing complex values can be stored but cannot be indexed when using table format. Either use fixed format, set index=False, or do not include the columns containing complex values to data_columns when initializing the table.rrUzcolumn z/ is not a data_column. In order to read column z: you must reload the dataframe into HDFStore and include z with the data_columns argument.)rr7rAr\r[rjr^r=rirrWrXZ remove_indexrrerrYrrq) rrrWrXkwrjrrArZ cur_optlevelZcur_kindrFrFrGrY+sR   zTable.create_indexrwz!list[tuple[ArrayLike, ArrayLike]]rrrc CsZt||||d}|}g}|jD]2}||j|j||j|j|jd}| |q"|S)a Create the axes sniffed from the table. Parameters ---------- where : ??? start : int or None, default None stop : int or None, default None Returns ------- List[Tuple[index_values, column_values]] r r/) Selectionrr7rrqrrrLrr) rr^rr selectionrLrruresrFrFrG _read_axess   zTable._read_axesrqcCs|S)zreturn the data for this objrFrrrqrFrFrG get_objectszTable.get_objectcst|s gS|d\}|j|i}|ddkrL|rLtd|d||dur^t}n |durjg}t|trt|t|}|fdd | Dfd d |DS) zd take the input data_columns and min_itemize and create a data columns spec rrr)z"cannot use a multi-index on axis [z] with data_columns TNcs g|]}|dkr|vr|qSrrFr)existing_data_columnsrFrGrZsz/Table.validate_data_columns..csg|]}|vr|qSrFrF)rWr) axis_labelsrFrGrZr) r]rqrrr[rAr9rCrGr)rrrrrrqrF)rrrGvalidate_data_columnss.     zTable.validate_data_columnsr&)rrUc/stts,|jj}td|dtddur:dgfddD|rzd}d d|jDt|j }|j }nd }|j } |j d ksJt |j d krtd g} |durd}fdddDd} j| } t| } |r|j|j|j|| ||-| |d }.t?|drl|j@|._@|.A||r|r|.B||.S)a0 Create and return the axes. Parameters ---------- axes: list or None The names or numbers of the axes to create. obj : DataFrame The object to create axes on. validate: bool, default True Whether to validate the obj against an existing object already written. nan_rep : A value to use for string column nan_rep. data_columns : List[str], True, or None, default None Specify the columns that we want to create to allow indexing on. * True : Use all available columns. * None : Use no columns. * List[str] : Use the specified columns. min_itemsize: Dict[str, int] or None, default None The min itemsize for a column in bytes. z/cannot properly create the storer for: [group->rtrNrcsg|]}|qSrF)_get_axis_numberrj)rrFrGrZrz&Table._create_axes..TcSsg|] }|jqSrFr?rjrFrFrGrZrFrDrUz.get_blk_items..)rrrFrrG get_blk_itemssz2Table._get_blocks_and_items..get_blk_itemsrr?cSs"i|]\}}t|||fqSrF)r\tolist)rWbrrFrFrGrBsz/Table._get_blocks_and_items..rcSsg|] }t|qSrFrK)rWitemrFrFrGrZrz/Table._get_blocks_and_items..z+cannot match existing table structure for [z] on appending data)rArr7rr r8r[rr]r(rHrQrGr&r\rLrPrrrrr)r{rrrrrrrrrrZ new_labelsrZby_itemsZ new_blocksZ new_blk_itemsZearrrr6ZjitemsrFrFrGrsN          zTable._get_blocks_and_itemsr)rc s|durt|}|durNjrNtjts.JjD]}||vr4|d|q4jD]\}}t|||qT|jdur|jD]$\}}fdd} | ||qS)zprocess axes filtersNrcsjD]}|}|}|dus*J||krfjrH|tj}||}j|d|S||vrtt |j }t|}t t rd|}||}j|d|Sqt d|ddS)Nr?rUzcannot find the field [z] for filtering!)Z _AXIS_ORDERSr _get_axisrunionr(rrOr6r^rLrAr&r)fieldfiltrZ axis_numberZ axis_valuesZtakersrLroprrFrGprocess_filter s"       z*Table.process_axes..process_filter) r[rrArinsertrrfilterr) rrrrrrlabelsrrrrFrrG process_axess  ! zTable.process_axes)rrr8rcCs|durt|jd}d|d}dd|jD|d<|rj|durH|jpFd}tj|||pZ|jd }||d <n|jdur~|j|d <|S) z:create the description of the table from the axes & valuesNi'rj)rPr8cSsi|]}|j|jqSrF)rrrjrFrFrGrB>rz,Table.create_description..r )rrrr)maxrr7rrtrrr)rrrrr8r:rrFrFrGcreate_description/s       zTable.create_descriptionrXc Cs|||sdSt||||d}|}|jdur|jD]D\}}}|j|||dd} ||| j |||j }qBt |S)zf select coordinates (row numbers) from a table; return the coordinates object Fr NrUrX) rWrr select_coordsrrrrrilocrLr() rr^rrrZcoordsrrrrlrFrFrGr Ns   zTable.read_coordinatesrcCs||sdS|dur$td|jD]z}||jkr*|jsNtd|dt|jj |}| |j |j ||||j |j|jd}tt|d|j|dSq*td|d dS) zj return a single column from the table, generally only indexables are interesting FNz4read_column does not currently accept a where clausezcolumn [z=] can not be extracted individually; it is not data indexabler/rUrOz] not found in the table)rWrrr7rPrrr^rjr=rrqrrrLrr+rrr)rrr^rrrurZ col_valuesrFrFrGrhs*      zTable.read_column)NrnNNNNNN)N)NNN)NN)TNNN)N)NNN)NNN)4rbrcrdrr?r@rrr'rrrrrrUrrrrrTrjrrr7rrrrrrrrrrRrSrWrrrrYrr9rrr staticmethodrrrr rr:rFrFrrGr s   "         KU"*k>: rc@s.eZdZdZdZd dddddZdd ZdS) rz a write-once read-many table: this format DOES NOT ALLOW appending to a table. writing is a one-time operation the data are stored in a format that allows for searching the data on disk rNrwrXcCs tddS)z[ read the indices and the indexing array, calculate offset rows and return z!WORMTable needs to implement readNrYrZrFrFrGrs zWORMTable.readcKs tddS)z write in a format that we can search later on (but cannot append to): write out the indices and the values using _write_array (e.g. a CArray) create an indexing table so that we can search z"WORMTable needs to implement writeNrYr[rFrFrGrszWORMTable.write)NNNN)rbrcrdrrwrrrFrFrFrGrs rc @sVeZdZdZdZdddZddd d d d Zd ddddddZddddddZdS)r(support the new appendable table formatsZ appendableNFTcCs|s|jr|j|jd|j||||| | d}|jD] }|q6|js|j|||| d}|| |d<|jj |jfi||j |j _ |jD]}| ||q|j || ddS)Nrj)r7rrUrrr)rrrr8r+)r)rrrrrr7rrrRZ create_tablerqrr write_data)rrr7rrrrrrr8rrrr+rjruoptionsrFrFrGrs4     zAppendableTable.writerwry)rrcs|jj}|j}g}|rT|jD]6}t|jjdd}t|tj r| |j dddqt |r|d}|ddD] }||@}qp| }nd}dd |jD} t | } | dksJ| d d |jD} d d | D} g} t| D]6\} }|f|j|| | j}| | | |q|dur$d }tjt|||jd }||d}t|D]x} | |t| d||kr|q|j|fdd | D|dur|ndfdd | DdqNdS)z` we form the data into a 2-d including indexes,values,mask write chunk-by-chunk rr?u1Fr-rUNcSsg|] }|jqSrF)rrjrFrFrGrZrz.AppendableTable.write_data..cSsg|] }|qSrF)rrjrFrFrGrZrc Ss,g|]$}|tt|j|jdqSr)Z transposerBZrollrrEr(rFrFrGrZrrr%csg|]}|qSrFrFrjZend_iZstart_irFrGrZ)rcsg|]}|qSrFrFr(r rFrGrZ+r)indexesr8rL)rrrrr.rlr;rArBrrr4r]r2rrrreshaperrrDwrite_data_chunk)rrrrrmasksrur8mr nindexesrLbvaluesrrAZ new_shaperowschunksrFr rGrsL        zAppendableTable.write_datarzlist[np.ndarray]znp.ndarray | None)rr r8rLc Cs|D]}t|jsdSq|djd}|t|krFtj||jd}|jj}t|}t|D]\} } | ||| <q^t|D]\} }|||| |<q||dur|j t dd} | s|| }t|r|j ||j dS)z Parameters ---------- rows : an empty memory space where we are putting the chunk indexes : an array of the indexes mask : an array of the masks values : an array of the values Nrr%Fr-)rBrrr]rrrrr2r4ryr;rjrr) rrr r8rLrArrrrrrrFrFrGr .s&   z AppendableTable.write_data_chunkrXcCsb|dust|sf|dur:|dur:|j}|jj|jddn(|durH|j}|jj||d}|j|S|srdS|j}t ||||d}| }t | }t|} | r^| } t| | dkj} t| sdg} | d| kr| | | ddkr| dd| } t| D]@} |t| | }|j||jd||jddd| } q|j| S)NTr/rXrUrr+)r]rrrrrjZ remove_rowsrrrrr+Z sort_valuesdiffr[rrrrPreversedrKrD)rr^rrrrjrrLZ sorted_serieslnrrZpgrrrFrFrGr5ZsD       zAppendableTable.delete) NFNNNNNNFNNT)F)NNN) rbrcrdrrwrrr r5rFrFrFrGrs$ <;,rc@s`eZdZUdZdZdZdZeZde d<e ddd d Z e dd d d Z ddddddZdS)rrrxrrDrArBryrcCs|jdjdkS)NrrU)rrrrFrFrGrsz"AppendableFrameTable.is_transposedrcCs|r |j}|S)zthese are written transposed)rsrrFrFrGrszAppendableFrameTable.get_objectNrwrXcs0|sdSj|||d}tjrHjjddini}fddtjD}t|dkstJ|d}||d} g} tjD]N\} } | j vrq|| \} }|ddkrt | }n t | }|d}|dur|j |d d jr |}|}t | t| d dd }n|j}t | t| d dd }|}|jdkrlt|tjrl|d|jdf}t|tjrt|j||d }n.t|t rt|||d }ntj|g||d }|j|jksJ|j|jf| |qt| dkr| d}n t| dd}t|||d}j|||d}|S)Nr rcs"g|]\}}|jdur|qSrr)rWrrrrFrGrZrz-AppendableFrameTable.read..rUrr)rTZinplacerPrOrr?)rr) rWrrr]rrqrrr7rr(r) from_tuples set_namesrr^rsrErArBrr rr&Z _from_arraysZdtypesrr;rr-rr)rr^rrrrkrqZindsindrframesrruZ index_valsrr=rrLZindex_Zcols_rrrFrrGrsZ        "   zAppendableFrameTable.read)NNNN)rbrcrdrr?rwrEr&rBrrrr9rrrFrFrFrGrs  rcsneZdZdZdZdZdZeZe ddddZ e dd d d Z dfd d Z dddddfdd ZZS)rrr~rrDryrcCsdSrrFrrFrFrGrsz#AppendableSeriesTable.is_transposedrcCs|SrIrFrrFrFrGrsz AppendableSeriesTable.get_objectNc s<t|ts|jpd}||}tjf||jd|S)+we are going to write this as a frame tablerLrr)rAr&rPZto_framerrrr)rrrrrPrrFrGr s   zAppendableSeriesTable.writerwr+rcs|j}|durB|rBt|jts"J|jD]}||vr(|d|q(tj||||d}|rj|j|jdd|jdddf}|j dkrd|_ |S)NrrTrrL) rrArr[rrr set_indexrrP)rr^rrrrrrErrFrGrs   zAppendableSeriesTable.read)N)NNNN)rbrcrdrr?rwrEr+rBrrr9rrrr:rFrFrrGrs rcs(eZdZdZdZdZfddZZS)rrr~rc s^|jpd}||\}|_t|jts*Jt|j}||t||_tj fd|i|S)rrLr) rPrrrAr[rr(rrr)rrrrPZnewobjr=rrFrGr2s    z AppendableMultiSeriesTable.write)rbrcrdrr?rwrr:rFrFrrGr,src@sdeZdZUdZdZdZdZeZde d<e ddd d Z e d d Z d dZ eddZddZdS)rz:a table that read/writes the generic pytables table formatrxryrDzlist[Hashable]rrNrcCs|jSrI)r?rrFrFrGr[FszGenericTable.pandas_typecCst|jddp|jSrrrrFrFrGrTJszGenericTable.storablecCsLg|_d|_g|_dd|jD|_dd|jD|_dd|jD|_dS)roNcSsg|]}|jr|qSrFrrjrFrFrGrZTrz*GenericTable.get_attrs..cSsg|]}|js|qSrFrrjrFrFrGrZUrcSsg|] }|jqSrFrOrjrFrFrGrZVr)rrrrrrrrrFrFrGrSNs zGenericTable.get_attrsc Cs|j}|d}|durdnd}tdd|j||d}|g}t|jD]^\}}t|tsZJt||}||}|durzdnd}t |||g||j||d} | | qD|S)z0create the indexables from the table descriptionrNrr)rPrrjrr)rPrrLrrjrr) rrrrjrZ_v_namesrArNr^r=r) rr:rrrrrrrrSrFrFrGrXs.     zGenericTable.indexablescKs tddS)Nz cannot write on an generic tablerYr[rFrFrGr{szGenericTable.writeN)rbrcrdrr?rwrEr&rBrrr[rTrSrrrrFrFrFrGr=s    "rcs`eZdZdZdZeZdZe dZ e ddddZ dfd d Z dd d d fdd ZZS)rza frame with a multi-indexrrDz ^level_\d+$rNrcCsdS)NZappendable_multirFrrFrFrGrsz*AppendableMultiFrameTable.table_type_shortNc sx|durg}n|dur |j}||\}|_t|jts@J|jD]}||vrF|d|qFtjf||d|S)NTrr) rrrrrAr[rrr)rrrrrrrFrGrs  zAppendableMultiFrameTable.writerwrXcsDtj||||d}|j}|jfdd|jjD|_|S)Nrcs g|]}j|rdn|qSrI) _re_levelssearch)rWrPrrFrGrZrz2AppendableMultiFrameTable.read..)rrrrrrr)rr^rrrrrrrGrs  zAppendableMultiFrameTable.read)N)NNNN)rbrcrdrrwr&rBrErecompilerrrrrr:rFrFrrGrs rr&r()rrrrcCs||}t|}|dur"t|}|dus4||rB||rB|St|}|durlt|j|dd}||stddg|j}|||<|jt|}|S)NF)sort) rr6equalsuniquerNslicerErOr\)rrrrrZslicerrFrFrGrs   rrz str | tzinfo)rrcCst|}|S)z+for a tz-aware type, return an encoded zone)rZ get_timezone)rzonerFrFrGrs rznp.ndarray | Indexzstr | tzinfo | Noneznp.ndarray | DatetimeIndex)rLrr"rcCst|tr"|jdus"|j|ks"J|durtt|trB|j}|j}n d}|}t|}t||d}|d|}n|rt j |dd}|S)a coerce the values to a DatetimeIndex if tz is set preserve the input shape if possible Parameters ---------- values : ndarray or Index tz : str or tzinfo coerce : if we do not have a passed timezone, coerce to M8[ns] ndarray NrOrgM8[ns]r%) rAr'rrPrr2rHrirjrBr0)rLrr"rPrFrFrGrs   r)rPrrLrrc Cstt|tsJ|j}t|\}}t|}t|}t|tsFt|j rlt ||||t |ddt |dd|dSt|t r~t dtj|dd} t|} | dkrtjdd | Dtjd }t ||dt|d S| d krt| ||}|j j} t ||d t| |d S| d vr$t |||||dSt|tjr>|j tksBJ|dksTJ|t}t |||||d SdS)Nrr)rLrXrrrrzMultiIndex not supported here!FrrcSsg|] }|qSrF) toordinalr(rFrFrGrZrz"_convert_index..r%)rr)integerZfloating)rLrXrrr1)rArNrPrrr;rr/r$rrr^r)rrrrBr0Zint32rtZ Time32Col_convert_string_arrayrrrr1r) rPrrLrrr7rrXrrrLrrFrFrGr}sT            r})rXrLrrcCs|dkrt|}n|dkr$t|}n|dkrvztjdd|Dtd}Wqtyrtjdd|Dtd}Yq0nT|dvrt|}n@|d vrt|d||d }n&|d krt|d }ntd ||S)Nr r#rcSsg|]}t|qSrFr&r(rFrFrGrZ6rz$_unconvert_index..r%cSsg|]}t|qSrFr)r(rFrFrGrZ8r)r*floatrr/r1rzunrecognized index type )r'r,rBr0r1rr6)rlrXrLrrrFrFrGr-s&     rrr)rPrrcCs|jtkr|Sttj|}|jj}tj|dd} | dkrBtdn&| dkrTtdn| dksh|dksh|St |} | } || | <tj| dd} | dkrt | j d D]V} | | } tj| dd} | dkrt || kr|| nd | }td |d | d qt| ||| j }|j}t|trBt||p>|dp>d }t|pLd |}|dur~||}|dur~||kr~|}|jd|dd}|S)NFrrz+[date] is not implemented as a table columnr]z>too many timezones in this block, create separate data columnsrr1rzNo.zCannot serialize the column [z2] because its data contents are not [string] but [z] object dtyperLz|Sr-)rr1r rBrrPrrrr.rmrDrr]r+r rrAr9rRrrrr4)rPrrrrrLrrrrr8rlrrZerror_column_labelrrZecirFrFrGrFsN       rr)rlrLrrcCs\t|r(t|j||j|j}t|}t dt |}t j |d|d}|S)a Take a string-like that is object dtype and coerce to a fixed size string type. Parameters ---------- data : np.ndarray[object] encoding : str errors : str Handler for encoding errors. Returns ------- np.ndarray[fixed-length-string] rUSr%)r]r+r2rNencoder5r rrr libwritersmax_len_string_arrayrBr0)rlrLrensuredrrFrFrGr+s r+cCs|j}tj|td}t|rvtt|}d|}t |dt r^t |j j ||dj}n|j|ddjtdd}|durd}t||||S) a* Inverse of _convert_string_array. Parameters ---------- data : np.ndarray[fixed-length-string] nan_rep : the storage repr of NaN encoding : str errors : str Handler for encoding errors. Returns ------- np.ndarray[object] Decoded data. r%Ur)rFr-Nr)rrBr0r2r1r]r/r0rrArlr+rNrDr5r4Z!string_array_replace_from_nan_repr )rlrrLrrrrrFrFrGr6s  r6)rLrrLrcCs6t|tsJt|t|r2t|||}||}|SrI)rArNr _need_convert_get_converter)rLrrLrconvrFrFrGrs  rrXrLrcs8|dkrddS|dkr&fddStd|dS)Nr cSstj|ddS)Nr(r%)rBr0rrFrFrGrrz _get_converter..rcst|ddS)Nr/)r6r7r}rFrGrsz invalid kind )rr6rFr}rGr4s r4rcCs|dvr dSdS)N)r rTFrFrrFrFrGr3sr3z Sequence[int])rPrErcCslt|tst|dkrtd|ddkrh|ddkrh|ddkrhtd|}|rh|d}d|}|S) z Prior to 0.10.1, we named values blocks like: values_block_0 an the name values_0, adjust the given name if necessary. Parameters ---------- name : str version : Tuple[int, int, int] Returns ------- str z6Version is incorrect, expected sequence of 3 integers.rrUrCrDzvalues_block_(\d+)Zvalues_)rArNr]rr!r r)rPrErgrprFrFrGrs$   r) dtype_strrcCst|}|ds|dr"d}n|dr2d}n|drBd}n|dsV|dr\d}nn|drld}n^|d r|d }nN|d rd }n>|d rd }n.|d rd}n|dkrd}ntd|d|S)zA Find the "kind" string describing the given dtype name. rrlr,rrRrr*r  timedeltar#ryrrr1zcannot interpret dtype of [r)rHrer)r:rXrFrFrGr s.       rrcCsbt|tr|j}|jjdd}|jjdvr@t| d}nt|t rP|j }t|}||fS)zJ Convert the passed data into a storable form and a dtype string. rLr)rMr) rAr0r rrPrrXrBr0rr*r)rlrrFrFrGr*s    rc@s<eZdZdZdddddddZdd Zd d Zd d ZdS)rz Carries out a selection operation on a tables.Table object. Parameters ---------- table : a Table object where : list of Terms (or convertible to) start, stop: indices to start and/or stop selection Nrrw)rjrrcCsh||_||_||_||_d|_d|_d|_d|_t|r.t t t j |dd}|dksd|dkrt |}|jt jkr|j|j}}|durd}|dur|jj}t ||||_nVt|jjt jr|jdur||jks|jdur||jkrt d||_Wdn1s$0Y|jdurd|||_|jdurd|j\|_|_dS)NFrr*booleanrz3where must have index locations >= start and < stop)rjr^rr conditionrZtermsr#r!rrrrrBr0rZbool_rr issubclassrr*r3generateevaluate)rrjr^rrinferredrFrFrGrLsD     &   zSelection.__init__c Cs|dur dS|j}zt|||jjdWSty|}z:d|}td|d|d}t||WYd}~n d}~00dS)z'where can be a : dict,list,tuple,stringN)rrLrz- The passed where expression: a* contains an invalid variable reference all of the variable references must be a reference to an axis (e.g. 'index' or 'columns'), or a data_column The currently defined references are: z ) rjrr3rL NameErrorrrrr)rr^rr6ZqkeysrrFrFrGr@{s  zSelection.generatecCsX|jdur(|jjj|j|j|jdS|jdurB|jj|jS|jjj|j|jdS)( generate the selection NrX) r>rjZ read_whererrrr#r rrrFrFrGrs  zSelection.selectcCs|j|j}}|jj}|dur$d}n|dkr4||7}|durB|}n|dkrR||7}|jdurx|jjj|j||ddS|jdur|jSt ||S)rDNrT)rrr#) rrrjrr>Zget_where_listrr#rBr)rrrrrFrFrGrs"  zSelection.select_coords)NNN)rbrcrdrrr@rrrFrFrFrGr@s/ r) ruNNFNTNNNNrnr@) NrrnNNNNFN)N)F)r __future__r contextlibrrmr]rrr$rr!textwraprtypingrrr r r r r rZnumpyrBZpandas._configrrZ pandas._libsrrr/Zpandas._libs.tslibsrZpandas._typingrrrZpandas.compat._optionalrZpandas.compat.pickle_compatrZ pandas.errorsrZpandas.util._decoratorsrZpandas.util._exceptionsrZpandas.core.dtypes.commonrrrrrr r!r"r#r$Zpandas.core.dtypes.missingr%rr&r'r(r)r*r+r,r-r.Zpandas.core.apir/Zpandas.core.arraysr0r1r2Zpandas.core.commoncorecommonr3Z pandas.core.computation.pytablesr3r4Zpandas.core.constructionr5Zpandas.core.indexes.apir6Zpandas.core.internalsr7r8Zpandas.io.commonr9Zpandas.io.formats.printingr:r;rpr<r=r>r?rOrJrHrMrQrVr_r2r`reWarningrfrrgrrhZ duplicate_docrrrrFZ dropna_docZ format_docZ config_prefixZregister_optionZis_boolZis_one_of_factoryrorsrtrrrrrrrrr;r=r>r\rrrrrrrrrrrrrrr}rrr+r6rr4r3rrrrrFrFrFrGs0   $       0 ,         &*:Np'1d _ifd1B+ (=K' !