Package yapydata :: Package datatree :: Module datatree

Source Code for Module yapydata.datatree.datatree

   1  # -*- coding: utf-8 -*- 
   2  """The *YapyData.datatree* module provides the core class for the handling of 
   3  simple data classes for the most common data representation languages. The 
   4  provided core class *DataTree* provided the *Python* data types, which comprise 
   5  the most standard data types. 
   6   
   7  The derived classes support for the most common data description languages 
   8  and add therefore the specific constraints and extensions. The package provides 
   9  these features for low-level libraries of the software stack, therefore depends  
  10  whenever possible on standard libraries only. It supports low-level read-only access 
  11  to files and in-memory data. The read data could be modified in-memory only for  
  12  example in order to superpose higher priority data read from the call options 
  13  of the command line . 
  14   
  15  The internal representation of the DDLs is exclusively compatible to the standard 
  16  *json* package in accordance to *RFC-7159*. The supported DDLs of the read files are: 
  17   
  18  * *INI* 
  19  * *JSON* 
  20  * *XML* 
  21  * *YAML* 
  22   
  23  A similar package for higher application layer levels is available by *multiconf*, 
  24  which provides sophisticated features such as cross-conversion and mixed-mode applications 
  25  by modularization, and enhanced processing plugins for various DDLs.     
  26   
  27  The validation and preparation including cross-conversion is supported by 
  28  **multiconf**, while the specifics of the *DL* is supported by the language 
  29  module such as *jsondata* and *xmldata*. 
  30  """ 
  31   
  32  from yapydata.datatree import YapyDataTreeError 
  33   
  34  import os 
  35   
  36  try: 
  37      import cPickle as pickle  # @UnusedImport 
  38  except: 
  39      import pickle  # @Reimport 
  40   
  41  from pythonids import ISSTR 
  42   
  43   
  44  __author__ = 'Arno-Can Uestuensoez' 
  45  __license__ = "Artistic-License-2.0 + Forced-Fairplay-Constraints" 
  46  __copyright__ = "Copyright (C) 2019 Arno-Can Uestuensoez" \ 
  47                  " @Ingenieurbuero Arno-Can Uestuensoez" 
  48  __version__ = '0.1.1' 
  49  __uuid__ = "60cac28d-efe6-4a8d-802f-fa4fc94fa741" 
  50   
  51  __docformat__ = "restructuredtext en" 
52 53 54 -class YapyDataDataTreeError(YapyDataTreeError):
55 """Common access error. 56 """ 57 pass
58
59 60 -class YapyDataTypeError(YapyDataTreeError):
61 """Common access error. 62 """ 63 pass
64
65 -def readout_data(xval, **kargs):
66 """For API call-compliance with other syntaxes. Returns here the 67 input tree only. 68 69 Args: 70 xval: 71 The input tree from the *DataTree*. 72 73 Returns: 74 The returns here the input *xval*. 75 76 Raises: 77 pass-through 78 79 """ 80 return xval
81
82 83 -class YapyDataDataTreeOidError(YapyDataDataTreeError):
84 """Requested object name is not present. 85 """
86 - def __init__(self, message='', *args, **kargs):
87 """Displays the issue of the exception. 88 89 Args: 90 message: 91 The message to be displayed. 92 Addition options *pathhook*, *path*, and 93 *trailer* are appended when present. 94 These are also provided as memmeber variables 95 for derived exceptions. 96 97 kargs: 98 pathhook: 99 The missing item of the path. 100 101 path: 102 Resolved path. 103 104 searchpath: 105 Optional search path. 106 107 trailer: 108 Optional textual hint. 109 110 Returns: 111 the raised exception 112 113 Raises: 114 itself 115 """ 116 self.message_in = message 117 message_out = message[:] 118 119 try: 120 self.pathhook = kargs.pop('pathhook') 121 except KeyError: 122 self.pathhook = '' 123 124 try: 125 self.path = kargs.pop('path') 126 except KeyError: 127 self.path = '' 128 try: 129 self.searchpath = kargs.pop('searchpath') 130 except KeyError: 131 self.searchpath = '' 132 try: 133 self.trailer = kargs.pop('trailer') 134 except KeyError: 135 self.trailer = '' 136 137 if self.pathhook or self.path: 138 message_out += "Missing subpath hook:" 139 140 if self.pathhook: 141 message_out += "\n pathhook: %s" % (str(self.pathhook)) 142 143 if self.path: 144 message_out += "\n path: %s" % (str(self.path)) 145 146 if self.searchpath: 147 message_out += "\n searchpath: %s" % (str(self.searchpath)) 148 149 if self.trailer: 150 message_out += str(self.trailer) 151 152 self.message_out = message_out 153 154 super(YapyDataDataTreeOidError, self).__init__(message_out, *args, **kargs)
155 156 157 # mode 158 M_COMP = 1 #: complete 159 M_FRAG = 2 #: fragment 160 M_INC = 4 #: increment 161 162 163 # strategy 164 S_CREA = 256 #: create 165 S_DEL = 512 #: delete 166 S_JOIN = 1024 #: join 167 S_MOD = 2048 #: modify 168 S_REP = 4096 #: replace 169 170 171 _debug = 0 172 _verbose = 0
173 174 175 -def grow_branch(*subpath, **kargs):
176 """Creates a new branch including the assigned value to 177 the last node. The node types are defined by the types 178 of the *subpath* entries. 179 180 Supports a single linear branch only, no sub-branching. 181 182 The created path is validated for permitted types. 183 The derived types such as JSON have to support their 184 own branch method. Thus provided as a static method. 185 186 Args: 187 subpath: 188 Variable list/tuple of path keys and indexes. 189 190 kargs: 191 value: 192 Value to be assigned to the final node. 193 194 default := None 195 196 Returns: 197 A created branch. 198 199 Raises: 200 YapyDataDataTreeOidError 201 202 pass-through 203 204 """ 205 _val = kargs.get('value') 206 _subpath=list(subpath) 207 try: 208 ik = _subpath.pop(0) 209 except IndexError: 210 return _val 211 212 if isinstance(ik, (bool, float, frozenset)) or ik == None or isinstance(ik, ISSTR): 213 # python only: (True, False,), None, ... 214 return {ik: grow_branch(*_subpath, value=_val)} 215 216 elif isinstance(ik, int): 217 if ik != 0: 218 # no padding 219 raise YapyDataDataTreeOidError( 220 "new list requires idx==0: %s\n see: %s\n" %( 221 str(subpath), 222 str(ik) 223 ) 224 ) 225 return [grow_branch(*_subpath, value=_val)] 226 227 raise YapyDataDataTreeOidError( 228 "invalid subpath key/index: %s\n see: %s\n" %( 229 str(subpath), 230 str(ik) 231 ) 232 )
233
234 235 -class DataTree(object):
236 """Provides JSON based read-only configuration of capabilities. 237 238 The access to structured data trees offers various methods to 239 access paths of nested node attributes. This comprises the 240 creation as well as the readout. 241 242 The following equivalent creation methods are supported, where 243 'treenode' could be either the root node, or any subordinated 244 branch:: 245 246 treenode['subnode0']['subnode1']['subnode7'] = value # dynamic items 247 248 value = treenode.create( # dynamic items by 'create()' 249 'subnode0', 'subnode1', 'subnode7', 250 ) 251 252 value = treenode.subnode0.subnode1.subnode7 # static attribute addressing style 253 254 The following equivalent readout methods are supported, where 255 'treenode' could be either the root node, or any subordinated 256 branch:: 257 258 value = treenode['subnode0']['subnode1']['subnode7'] # dynamic items 259 value = treenode('subnode0', 'subnode1', 'subnode7') # dynamic items by '__call__' 260 value = treenode.subnode0.subnode1.subnode7 # static attribute addressing style 261 262 """ 263 264 M_FIRST = 1 # use first matching node 265 M_LAST = 2 # use last matching node 266 M_ALL = 3 # use all - iterate all matches 267 268 match_map = { 269 M_FIRST: 1, 270 M_LAST: 2, 271 M_ALL: 3, 272 'first': 1, 273 'last': 2, 274 'all': 3, 275 } 276 277 @staticmethod
278 - def isvalid_top(value, **kargs):
279 """Validate compliance of top-node. To be provided by derived classes 280 for specific syntaxes. 281 282 Args: 283 value: 284 Top node. 285 286 kargs: 287 Specific syntax related dynamic parameters to be defined 288 by derived classes. 289 290 Returns: 291 None 292 293 Results: 294 YapyDataTreeError: 295 Raises exception when not valid. 296 297 """ 298 pass
299
300 - def __init__(self, data=None):
301 """ 302 Args: 303 data: 304 Configuration data in accordance to the selected data language syntax. 305 The default is the Python syntax including the appropriate data types. 306 This may impose additional constraints by derived classes e.g. in case 307 of persistent data such as JSON and XML - see other classes within 308 this module. 309 310 The default Python DL implementation supports in-memory access only, 311 while persistence will be available for example by pickling. 312 313 The initial *data* defines the permitted type of the first item 314 within the *subpath* of the spanned data tree. The default value acts 315 here as a placeholder for an empty structure, which could be defined 316 by following extension operations arbitrarily. 317 318 The basic constraint introduced here is that intermediate nodes 319 require support of subscription. This is due to the addressing concepts 320 implemented by *DataTree*. Thus even though a *set* could technically 321 be an intermediate node, it could not be indexed, thus could not be 322 addressed by the standard member functions. Resulting 'set' and 'frozenset' 323 are supported by *DataTree* as end-nodes only, same as atomic data types. 324 325 Anyhow, the <top-node> is by default permitted to be an end-node. Thus 326 the context defines the applicability dynamically. 327 328 The consistency of the data tree including the valid intermediate nodes 329 is verified by access, so basically within the responsibility of the caller. 330 331 Returns: 332 333 None / initialized object 334 335 Raises: 336 YapyDataDataTreeError 337 338 pass-through 339 340 """ 341 self.data = data
342
343 - def __call__(self, *subpath, **kargs):
344 """Readout the value of a node, or an attribute. The name binding 345 of the path is provided as a tuple of path items. 346 347 Args: 348 subpath: 349 The list of keys constituting a branch of a data tree. 350 The *subpath* is treated as a branch of one of the nodes 351 of a provided *searchpath* - which is by default the top node. 352 The supported values are:: 353 354 subpath := <list-of-node-ids> 355 <list-of-node-ids> := <node-id> [',' <list-of-node-ids>] 356 node-id := ( 357 str # string: dict 358 | int # integer: lists, tuple, dict 359 ) 360 361 The value of the node within *data*:: 362 363 nodeid := ( 364 <single-nodeid> 365 | <list-of-nodeids> 366 | <tuple-of-nodeids> 367 ) 368 single-nodeid := <nodeid> 369 list-of-nodeids := '[' <nodeidlists> ']' 370 tuple-of-nodeids := '(' <nodeidlists> ')' 371 nodeidlists := <nodeid> [',' <nodeidlists>] 372 nodeid := ( 373 ItemKey 374 | ListIndex 375 ) 376 ItemKey := "valid dict-key" 377 ListIndex := "valid list-index" 378 379 The derived syntax classes may impose specific constraints. 380 Thus it is recommended to use integers and strings only 381 for maximum compatibility, and the ease of using mixed syntaxes:: 382 383 ItemKey := str # string: dict 384 ListIndex := int # integer: lists, tuple, dict 385 386 kargs: 387 searchpath: 388 Optional search path for the match of the provided 389 address *subpath*. The provided *subpath* is applied 390 to each node of the *searchpath* in accordance to the 391 *direction* option. This provides the search and 392 enumeration of side branches:: 393 394 searchpath := <path-item-list> 395 396 path-item-list := <path-item> [, <path-item-list>] 397 path-item := ( 398 str # item name 399 | int # item index 400 ) 401 402 default := <top-node> 403 404 The search path entries has to be actually present by default. 405 These could be either created by loading a complete tree 406 structure, or by using the *Capabilities.create()* member. 407 See also parameter 'strict'. 408 409 direction: 410 The search direction of the *subpath* within the 411 *searchpath*. In case of multiple superpositioned 412 attributes the first traversed match. 413 414 The provided values are:: 415 416 direction := ( 417 up | 0 | False # search from right-to-left 418 | down | 1 | True # search from left-to-right 419 ) 420 421 default:= up 422 423 match: 424 Sets the match criteria for the search operation. 425 Interferes with *direction*:: 426 427 match := ( 428 M_FIRST | 'first' # use first matching node 429 | M_LAST | 'last' # use last matching node 430 | M_ALL | 'all' # use all - iterate all matches 431 ) 432 433 default := M_FIRST 434 435 partial: 436 Enables the return of partial sub paths in case the requested 437 path is not completely present. :: 438 439 partial := ( 440 True # when not completely present, the longest 441 # existing part is returned, the completeness 442 # is provided by the result attribute <partial> 443 | False # when not completely present an exception 444 # is raised 445 ) 446 447 strict: 448 Controls the required consistency. This comprises: 449 450 1. the presence of the search path entries 451 452 2. the presence of the requested subpath within the set 453 of search paths 454 455 pysyn: 456 Activates full *Python* syntax. This in particular enables all 457 container types of intermediate nodes for arbitrary paths. 458 Includes *tuple*, *set*, *frozenset*, etc. :: 459 460 pysyn := ( 461 True # allows all python types as container nodes 462 | False # allows list and dict only as container nodes 463 ) 464 465 default := False 466 467 Returns: 468 In case of a match returns the tuple:: 469 470 return := (<attr-value-path>, <attr-value>, <partial>) 471 472 attr-value-path := ( 473 "the list of keys of the top-down path" 474 | "empty list when no item exists" # see <complete> 475 ) 476 attr-value := "value of the targeted node/attribute" 477 partial := ( 478 False # the complete requested path 479 | True # the actually present part of the path 480 ) 481 482 Else raises *YapyDataDataTreeOidError*. 483 484 Raises: 485 YapyDataDataTreeOidError 486 487 pass-through 488 489 """ 490 _srch = kargs.get('searchpath', ()) 491 _dir = kargs.get('direction', 0) 492 _match = kargs.get('match', DataTree.M_FIRST) 493 _pysyn = kargs.get('pysyn') 494 495 if not isinstance(_srch, (tuple, list,)): 496 raise YapyDataDataTreeError( 497 "search path requires 'tuple' or'list', got: " 498 + str(_srch) 499 ) 500 501 # 502 # match criteria 503 # 504 try: 505 _match = self.match_map[_match] 506 except IndexError: 507 try: 508 _match = self.match_map[str(_match).lower()] 509 except KeyError: 510 raise YapyDataDataTreeError( 511 "valid match are (first, %d, last, %d, all, %d,), got: %s" %( 512 DataTree.M_FIRST, 513 DataTree.M_LAST, 514 DataTree.M_ALL, 515 str(_match) 516 ) 517 ) 518 519 # 520 # search direction 521 # 522 if _dir in (True, False,): 523 pass 524 else: 525 _dir = str(_dir).lower() 526 if _dir in ('up', '0',): 527 _dir = False 528 elif _dir in ('down', '1',): 529 _dir = True 530 else: 531 raise YapyDataDataTreeError( 532 "valid directions are (up, 0, down, 1), got: " 533 + str(_dir) 534 ) 535 536 # collect the nodes on the searchpath 537 _path_nodes = [self.data,] 538 _cur = self.data 539 if _srch: 540 for x in _srch: 541 try: 542 _cur = _cur[x] 543 except (IndexError, KeyError, TypeError): 544 raise YapyDataDataTreeOidError( 545 "invalid search path: %s\n see: %s\n" %( 546 str(_srch), 547 str(x) 548 ) 549 ) 550 _path_nodes.append(_cur) 551 552 # revert for bottom-up search direction 553 if not _dir: 554 # upward - up | 0 | False 555 _path_nodes = reversed(_path_nodes) 556 557 # now search the subpath for each node of the search path 558 # first match wins 559 for _pn in _path_nodes: 560 _cur = _pn 561 _idx_subpath = 0 # reset here 562 for x in subpath: 563 _excep = False 564 try: 565 _cur = _cur[x] 566 567 except (IndexError, KeyError,): 568 # a valid type - but missing value 569 # continue with next level - only when nodes do not fit 570 _cur = None 571 _excep = True 572 break 573 574 except TypeError: 575 # not a valid data type 576 if _pysyn: 577 try: 578 _cur = getattr(_cur, x) 579 580 except TypeError: 581 if isinstance(_cur, set): 582 try: 583 _cur=_cur & set([x]) 584 _cur = list(_cur)[0] 585 586 except: 587 _cur = None 588 _excep = True 589 break 590 591 except AttributeError: 592 try: 593 _cur = _cur[int(x)] 594 595 except Exception as e: # for debug 596 _cur = None 597 _excep = True 598 break 599 600 except Exception as e: # for debug 601 _cur = None 602 _excep = True 603 break 604 else: 605 # continue with next level - only when nodes do not fit 606 _cur = None 607 _excep = True 608 break 609 610 if not _excep: 611 break # has hit a regular match 612 613 if _excep: 614 # no match - prefer a message with error hint here 615 raise YapyDataDataTreeOidError( 616 '', 617 pathhook=str(subpath[_idx_subpath - 1]), 618 path=str(subpath), 619 searchpath=str(_srch), 620 ) 621 622 return _cur
623 624 # def __str__(self): 625 # res = '' 626 # return res 627 628
629 - def create(self, *subpath, **kargs):
630 """Creates a *subpath* to a given node, default is from top. 631 Reuses existing nodes, starts the creation at the first point 632 of branch-out from the exiting tree. 633 634 In general no padding of pre-required entries is done. This e.g. 635 requires in case of a *list* the start with the index *0*, while 636 in case of the *dict* arbitrary keys could be assigned. 637 638 Args: 639 subpath: 640 The list of keys constituting a branch of a data tree. 641 The *subpath* is treated as a branch of one of the nodes 642 of a provided *searchpath* - which is by default the top node. 643 The supported values are:: 644 645 subpath := <list-of-node-ids> 646 <list-of-node-ids> := <node-id> [',' <list-of-node-ids>] 647 node-id := ( 648 <present-intermediate-node> 649 | <new-node> 650 ) 651 present-intermediate-node := ( 652 str # string: dict 653 | int # integer: lists, tuple, dict 654 | True | False # logic: dict 655 | None # null: dict 656 ) 657 new-node := ( 658 str # string: dict 659 | int # integer: list 660 | True | False # logic: dict 661 | None # null: dict 662 ) 663 664 Some *Python* data types are immutable, which could be subscripted read-only, 665 e.g. strings. While others such as sets are iterable, but not subscriptable 666 at all. Refer to the manual for a detailed list. 667 668 kargs: 669 hook: 670 Optional node as parent of the insertion point for the new sub path. 671 The node must exist and be part of the targeted data structure. No 672 additional checks are done:: 673 674 hook := <memory-node> 675 memory-node := "node address" 676 677 default := <top-node> 678 679 The *hook* node could either be a border node of the existing tree, 680 or any arbitrary node with a partial of complete part of the requested 681 *subpath*. Existing nodes are reused. 682 683 strict: 684 If *True* requires all present nodes of the *subpath* to of the 685 appropriate type, missing are created. When *False* present nodes 686 of inappropriate type are simply replaced. :: 687 688 strict := ( 689 True # nodes must be present 690 | False # missing are created 691 ) 692 default := False 693 694 Nodes of type *None* are always treated as non-present placeholder, 695 thus replaced in any case. 696 697 value: 698 Value of the created final node:: 699 700 value := <value-of-node> 701 702 value-of-node := <valid-json-node-type> 703 valid-json-node-type := ( 704 int | float 705 | str # unicode 706 | dict | list 707 | None | True | False # equivalent: null|true|false 708 ) 709 710 default := None 711 712 Returns: 713 In case of success the in-memory nodes of the sub path:: 714 715 return := (<attr-value-path>) 716 717 attr-value-path-tuple := ( 718 <in-memory nodes> 719 | <non-subscriptable-node> 720 ) 721 in-memory nodes := ( 722 "the list of in-memory nodes with support of subscription" 723 ) 724 non-subscriptable-node := "any valid type" 725 726 else raises *YapyDataDataTreeOidError*. 727 728 The last node contains in case of an atomic type the value 729 of the node, while the intermediate nodes represent the 730 indexed containers. 731 732 Raises: 733 YapyDataDataTreeOidError 734 735 pass-through 736 737 """ 738 739 _strict = kargs.get('strict', False) 740 _val = kargs.get('value', None) 741 742 if not subpath and isinstance(subpath, (tuple,)): 743 # no path supported at all - so it is a replacement value for self.data only 744 self.data = _val 745 return (self.data,) 746 747 # collect the nodes on the search path 748 # requires a stepwise lookahead, doing it recursive 749 _last = None # last successful container-node 750 _hook = kargs.get('hook', self.data) # the current insertion point 751 _path_nodes = [] 752 _subpath = list(subpath) 753 while _subpath: 754 try: 755 # iterate the present nodes 756 _hook = _hook[_subpath[0]] 757 _path_nodes.append(_hook) 758 _last = _subpath.pop(0) # store it for backlog of branch-out on non-container hook 759 760 # 761 # now work out and create missing nodes 762 # 763 except IndexError: 764 # it is a new item in a list, so to be appended - sparse in not permitted/supported 765 if _subpath[0] != len(_hook): 766 raise YapyDataDataTreeOidError( 767 '', 768 pathhook=str(_subpath[0]), 769 path=str(subpath), 770 ) 771 _k = _subpath.pop(0) 772 773 # now build the new part 774 _path_nodes.append(grow_branch(*_subpath, value=_val)) 775 _hook.append(_path_nodes[-1]) 776 return tuple(_path_nodes) 777 778 except KeyError: 779 # it is a new item in a 'dict' - just insert it 780 # now build the new part 781 _k = _subpath.pop(0) 782 _node = grow_branch(*_subpath, value=_val) 783 784 _hook[_k] = _node 785 _path_nodes.append(_node) 786 787 # add created subpaths for result vector 788 for _kx in _subpath: 789 _path_nodes.append(_path_nodes[-1][_kx]) 790 791 return tuple(_path_nodes) 792 793 except TypeError: 794 # it is a new item, but not within a container, 795 # so replaces if permitted for replacement, 796 # check of valid condition by: 797 # 798 # - _strict==False - replace as required if valid 799 # - None - placeholder for non-present 800 # 801 if _strict: 802 # at least one present node does not match required strict type-condition 803 # requires a container, got an atomic or non-indexable(set, ...) 804 raise YapyDataDataTreeOidError( 805 "invalid subpath: %s\n see: subpath[%s] = %s\n" %( 806 str(subpath), 807 str(subpath.index(_subpath[0])), # for robustness... 808 str(_subpath[0]) 809 ) 810 ) 811 812 if _last == None: 813 # so it was the first attempt 814 # 815 # here the self.data is the first created node, thus included in the return 816 # can check it by id(<self-obj>.data) == id(<return-val>[0]) 817 # 818 self.data = grow_branch(*_subpath, value=_val) 819 820 # add created subpaths for result vector 821 _k = _subpath.pop(0) 822 _path_nodes.append(self.data[_k]) 823 for _kx in _subpath: 824 _path_nodes.append(_path_nodes[-1][_kx]) 825 826 return tuple(_path_nodes) 827 828 829 #*** 830 #*** here we had a partial resolution with a trailing non-container node *** 831 #*** the node-value is released for replacement by strict==False *** 832 #*** 833 834 # no container type fixed yet, 835 # so now add, and drop/replace the non-container node 836 _k = _subpath.pop(0) 837 if isinstance(_k, int): 838 # key is 'int' so as defined the (default) container is a list 839 # 'int' keys of 'dict' are not supported for creation, while the read access is provided 840 # so create them manually, use them as you like 841 if _k == 0: # requires _k==0 because here it is the first 842 raise YapyDataDataTreeOidError( 843 "invalid subpath inital index range for new 'list': %s\n see: subpath[%s] = %s\n" %( 844 str(subpath), 845 str(subpath.index(_k)), # for robustness... 846 str(_k) 847 ) 848 ) 849 850 _path_nodes[-1] = _path_nodes[-2][_last] = [ grow_branch(*_subpath, value=_val), ] 851 return tuple(_path_nodes) 852 853 elif isinstance(_k, ISSTR): 854 # this basically should never fail - as long as '_k' is immutable... 855 # so for now want the eventual exception including it's stack... 856 if _path_nodes: 857 if len(_path_nodes) >1: 858 _path_nodes[-1] = _path_nodes[-2][_last] = { _k: grow_branch(*_subpath, value=_val)} 859 _path_nodes.append(_path_nodes[-1][_k]) 860 861 # add created subpaths for result vector 862 for _kx in _subpath: 863 _path_nodes.append(_path_nodes[-1][_kx]) 864 865 else: # ==1 866 self.data[_last] = {_k: grow_branch(*_subpath, value=_val)} 867 _path_nodes[-1] = self.data[_last] 868 _path_nodes.append(self.data[_last][_k]) 869 870 else: 871 self.data = {_k: grow_branch(*_subpath, value=_val)} 872 _path_nodes.append(self.data) 873 874 return tuple(_path_nodes) 875 876 else: 877 raise YapyDataDataTreeOidError( 878 "invalid subpath key/index type: %s\n see: subpath[%s] = %s\n" %( 879 str(subpath), 880 str(subpath.index(_k)), # for robustness... 881 str(_k) 882 ) 883 ) 884 885 # 886 # here we did not had any exception, that means the path is present, 887 # now let us check the value of the last item 888 # 889 try: 890 _path_nodes[-1] = _val 891 _path_nodes[-2][_last] = _val 892 except: 893 _path_nodes[-1] = self.data = _val 894 895 return tuple(_path_nodes)
896
897 - def get(self, *path):
898 """Gets the value of the path within the member 'data':: 899 900 self.data[key] 901 self.data[key0][key1][key2]... 902 903 904 Args: 905 key: 906 The value of the node within *data*:: 907 908 key := ( 909 <single-key> 910 | <list-of-keys> 911 | <tuple-of-keys> 912 ) 913 single-key := <key> 914 list-of-keys := '[' <keylists> ']' 915 tuple-of-keys := '(' <keylist> ')' 916 keylist := <key> [',' <keylist>] 917 key := ( 918 ItemKey 919 | ListIndex 920 ) 921 ItemKey := "valid dict-key" 922 ListIndex := "valid list-index" 923 924 Returns: 925 The value of the addressed node/value. 926 927 Raises: 928 pass-through 929 930 """ 931 """Gets the value of the path within the member 'data':: 932 933 self.data[nodeid] 934 self.data[nodeid0][nodeid1][nodeid2]... 935 936 When fails, a final trial is given to *list* and *dict* classes 937 including mixins:: 938 939 self[nodeid] 940 941 When finally still missing at all, an exception is raised. 942 943 Args: 944 nodeid: 945 The value of the node within *data*:: 946 947 nodeid := ( 948 <single-nodeid> 949 | <list-of-nodeids> 950 | <tuple-of-nodeids> 951 ) 952 single-nodeid := <nodeid> 953 list-of-nodeids := '[' <nodeidlists> ']' 954 tuple-of-nodeids := '(' <nodeidlists> ')' 955 nodeidlists := <nodeid> [',' <nodeidlists>] 956 nodeid := ( 957 ItemKey 958 | ListIndex 959 ) 960 ItemKey := "valid dict-key" 961 ListIndex := "valid list-index" 962 963 The derived syntax classes may impose specific constraints. 964 Thus it is recommended to use integers and strings only 965 for maximum compatibility, and the ease of using mixed syntaxes:: 966 967 ItemKey := str # string: dict 968 ListIndex := int # integer: lists, tuple, dict 969 970 Returns: 971 The value of the addressed node/value. 972 973 Raises: 974 pass-through 975 976 """ 977 try: 978 return self(*path) 979 except: 980 return None
981
982 - def import_data(self, fpname, key=None, node=None, **kargs):
983 """The core class *DataTree *does not support serialization. 984 985 For serialization use either a derived syntax class, or 986 serialize it e..g. by pickling and use the in-memory data. 987 For example by pickling:: 988 989 def import_data(self, fpname, key=None, node=None, **kargs): 990 if not os.path.isfile(fpname): 991 if not os.path.isfile(fpname + '.pkl'): 992 raise YapyDataTreeError( 993 "Missing file: " + str(fpname)) 994 else: 995 fpname = fpname + '.pkl' 996 997 if key and node == None: 998 raise YapyDataTreeError( 999 "Given key(%s) requires a valid node." % (str(key))) 1000 1001 datafile = os.path.abspath(fpname) 1002 with open(datafile, 'rb') as data_file: 1003 pval = pickle.load(data_file) 1004 1005 if key: 1006 node[key] = pval 1007 else: 1008 self.data = pval 1009 1010 return pval 1011 1012 See manuals for security issues of pickling. 1013 1014 """ 1015 raise NotImplementedError( 1016 """Use a derived syntax class, or serialize by pickler.""" 1017 )
1018
1019 - def join(self, data, keyidx=None, hook=None):
1020 """Superposes a JSON structure onto an existing. This 1021 is a fixed mode and strategy special case of the generic 1022 method *superpose()*. Implemented by recursion. The reduced 1023 parameter set provides better performance on large trees 1024 while the graph parameters still could be efficiently set 1025 by default values. 1026 1027 The superpositioning is supported by multiple strategies 1028 defined by the parameter *mode*. The provided algorithm 1029 of the strategy is *join*, where the input data is processed 1030 on the exisiting tree by modification and creation as required. 1031 1032 * branches are resolved from top to the leafs 1033 * missing sub-branches are created 1034 * missing leafs are created 1035 * existing leafs are replaced 1036 1037 This implements a last-wins strategy, thus in particular supports 1038 incremental load of configurations by raising priority. 1039 1040 Args: 1041 data: 1042 Top node of the data tree to be superposed. 1043 1044 keyidx: 1045 Key or index to be used at the insertion node. 1046 If not given the insertion node is:: 1047 1048 into dict: update by the new node 1049 into list: append the new node 1050 1051 default := None 1052 1053 hook: 1054 The insertion node for the new data:: 1055 1056 when not given: use top. 1057 1058 default := None 1059 1060 Returns: 1061 Returns the merged data tree, 1062 raises an exception in case of failure. 1063 1064 Raises: 1065 YapyDataDataTreeError 1066 1067 pass-through 1068 1069 """ 1070 if not data: 1071 return 1072 1073 if not hook: 1074 hook=self.data 1075 1076 if isinstance(hook, dict): 1077 1078 if isinstance(data, (int, float,)) or isinstance(data, ISSTR): # data is atom 1079 if not keyidx: 1080 raise YapyDataDataTreeError( 1081 "update dict item(%s) requires key" % (str(type(data))) 1082 ) 1083 1084 hook[keyidx] = data 1085 1086 elif isinstance(data, dict): # data is dict 1087 try: 1088 _hookx = hook[keyidx] 1089 except KeyError: 1090 _hookx = hook 1091 1092 if isinstance(_hookx, dict): 1093 # will use itself - including keys - for update 1094 for k,v in data.items(): 1095 1096 if _hookx.get(k): 1097 # replace an existing sub-tree 1098 self.join(v, k, _hookx) 1099 else: 1100 # add a new subtree 1101 _hookx[k] = v 1102 1103 elif isinstance(_hookx, list): 1104 1105 if keyidx == None: 1106 raise YapyDataDataTreeOidError( 1107 "Insertion into list requires index, got: " + str(keyidx) 1108 ) 1109 1110 elif isinstance(keyidx,ISSTR): 1111 try: 1112 hook[keyidx] = data 1113 except KeyError: 1114 raise YapyDataDataTreeOidError( 1115 "Insertion into or replacement of list failed: %s" % ( 1116 str(keyidx), 1117 ) 1118 ) 1119 hook[keyidx] = data 1120 1121 else: 1122 self.join(data, keyidx, hook) 1123 1124 else: # data is list or atom 1125 if not keyidx: # requires a key 1126 raise YapyDataDataTreeError( 1127 "cannot update dict with %s" % (str(type(data))) 1128 ) 1129 1130 else: 1131 if not hook.get(keyidx): 1132 # a new branch - yust add it 1133 hook[keyidx] = data 1134 1135 elif type(hook.get(keyidx)) != type(data): 1136 # icompatible types - for now replace 1137 hook[keyidx] = data 1138 1139 else: 1140 # superpose existing by leafs, missing by branches and/or leafs 1141 _hookx = hook[keyidx] 1142 for i in range(len(data)): 1143 if i < len(_hookx): 1144 # superpose an existing sub-tree 1145 self.join(data[i], i, _hookx) 1146 elif i == len(_hookx): 1147 # add a new subtree 1148 _hookx.append(data[i]) 1149 else: 1150 # add a new subtree 1151 raise YapyDataDataTreeOidError( 1152 "index(%s) out of range(%s). " % ( 1153 str(i), 1154 str(len(_hookx)), 1155 ) 1156 ) 1157 1158 elif isinstance(hook, list): 1159 if keyidx == None or keyidx == len(hook): 1160 # so fill incrementally by default 1161 hook.append(data) 1162 1163 elif keyidx > len(hook): 1164 # no sparse lists suported 1165 raise YapyDataDataTreeOidError( 1166 "Insertion index(%s) out of range(%s). " % ( 1167 str(len(hook)), 1168 str(keyidx), 1169 ) 1170 ) 1171 elif ( 1172 isinstance(data, (int, float,)) or isinstance(data, ISSTR) # data is atom 1173 or not isinstance(hook[keyidx], (dict, list,)) # target is atom 1174 ): 1175 if keyidx == None: 1176 raise YapyDataDataTreeError( 1177 "update dict item(%s) requires key" % (str(type(data))) 1178 ) 1179 1180 hook[keyidx] = data 1181 1182 else: 1183 # superpose an existing item 1184 self.join(data, keyidx, hook) 1185 1186 return
1187