-=[ Mr. Bumblebee ]=-
_Indonesia_
Qc @@ s d Z d d l m Z d d l Z d d l m Z e e d d d l m Z m Z m
Z
m Z m Z m
Z
m Z m Z m Z m Z d d l m Z m Z m Z d Z d Z d
Z d Z d Z d
Z d Z d Z d e f d YZ d e f d YZ d e f d YZ! d e j" f d YZ# d e$ f d YZ% d e f d YZ& d e f d YZ' e% Z( y d d l m) Z* e* j+ Z( Wn0 e, k
rZ- e j. e- d d l m/ Z* n Xd S( s B+Tree indicesi ( t absolute_importN( t lazy_imports7
import bisect
import math
import tempfile
import zlib
(
t chunk_writert debugt errorst
fifo_cachet indext lru_cachet osutilst static_tuplet tracet transport( t _OPTION_NODE_REFSt _OPTION_KEY_ELEMENTSt _OPTION_LENs B+Tree Graph Index 2
s row_lengths=s
type=leaf
s type=internal
s offset=ix i i t _BuilderRowc B@ s# e Z d Z d Z e d Z RS( s The stored state accumulated while writing out a row in the index.
:ivar spool: A temporary file used to accumulate nodes for this row
in the tree.
:ivar nodes: The count of nodes emitted so far.
c C@ s d | _ d | _ d | _ d S( s Create a _BuilderRow.i N( t nodest Nonet spoolt writer( t self( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt __init__D s c C@ s$ | j j \ } } } | j d k rM t j | _ | j j d t nC | j d k r t j d d } | j | j j
| | _ n d } | r | r | d =| } n | j j | | j j | t
} | d k rt d | j j | f n | j d 7_ d | _ d S( Ni t i t prefixs bzr-index-row-is incorrect node length: %d, %d( R t finishR t cStringIOt StringIOR t writet _RESERVED_HEADER_BYTESt tempfilet
TemporaryFilet getvaluet
writelinest tellt
_PAGE_SIZEt AssertionErrorR ( R t padt
byte_linest _t paddingR t
skipped_bytest remainder( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt finish_nodeJ s&
( t __name__t
__module__t __doc__R t TrueR* ( ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR < s t _InternalBuilderRowc B@ s e Z d Z e d Z RS( s= The stored state accumulated while writing out internal rows.c C@ s& | s t d n t j | d S( Ns Must pad internal nodes only.( R# R R* ( R R$ ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR* e s ( R+ R, R- R. R* ( ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR/ b s t _LeafBuilderRowc B@ s e Z d Z RS( s; The stored state accumulated while writing out a leaf rows.( R+ R, R- ( ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR0 k s t BTreeBuilderc B@ s e Z d Z d d d d Z d d Z d Z d Z d Z d Z d
Z d Z
e d Z e d
Z
d Z d Z d Z d Z d Z d Z d Z RS( s) A Builder for B+Tree based Graph indices.
The resulting graph has the structure:
_SIGNATURE OPTIONS NODES
_SIGNATURE := 'B+Tree Graph Index 1' NEWLINE
OPTIONS := REF_LISTS KEY_ELEMENTS LENGTH
REF_LISTS := 'node_ref_lists=' DIGITS NEWLINE
KEY_ELEMENTS := 'key_elements=' DIGITS NEWLINE
LENGTH := 'len=' DIGITS NEWLINE
ROW_LENGTHS := 'row_lengths' DIGITS (COMMA DIGITS)*
NODES := NODE_COMPRESSED*
NODE_COMPRESSED:= COMPRESSED_BYTES{4096}
NODE_RAW := INTERNAL | LEAF
INTERNAL := INTERNAL_FLAG POINTERS
LEAF := LEAF_FLAG ROWS
KEY_ELEMENT := Not-whitespace-utf8
KEY := KEY_ELEMENT (NULL KEY_ELEMENT)*
ROWS := ROW*
ROW := KEY NULL ABSENT? NULL REFERENCES NULL VALUE NEWLINE
ABSENT := 'a'
REFERENCES := REFERENCE_LIST (TAB REFERENCE_LIST){node_ref_lists - 1}
REFERENCE_LIST := (REFERENCE (CR REFERENCE)*)?
REFERENCE := KEY
VALUE := no-newline-no-null-bytes
i i i c C@ sM t j j | d | d | | | _ g | _ i | _ d | _ t | _ d S( s See GraphIndexBuilder.__init__.
:param spill_at: Optional parameter controlling the maximum number
of nodes that BTreeBuilder will hold in memory.
t reference_listst key_elementsN(
R t GraphIndexBuilderR t _spill_att _backing_indicest _nodesR t
_nodes_by_keyt Falset _optimize_for_size( R R2 R3 t spill_at( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s c C@ s t j j | j } | j | | | \ } } | | j k rW t j | | n t j | | | j | <| j d k r | j
d k r | j | | | n t | j | j
k r d S| j d S( s9 Add a node to the index.
If adding the node causes the builder to reach its spill_at threshold,
disk spilling will be triggered.
:param key: The key. keys are non-empty tuples containing
as many whitespace-free utf8 bytestrings as the key length
defined for this index.
:param references: An iterable of iterables of keys. Each is a
reference to another key.
:param value: The value to associate with the key. It may be any
bytes as long as it does not contain \0 or \n.
i N( R t StaticTuplet
from_sequencet internt _check_key_ref_valueR7 R t BadIndexDuplicateKeyR8 R t _key_lengtht _update_nodes_by_keyt lenR5 t _spill_mem_keys_to_disk( R t keyt valuet
referencest node_refsR& ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt add_node s c C@ s | j r! | j \ } } } n | j \ } } t t j d d | } | | _ | j r t | j | k r | j j d n | | j | <x4 t | D] } d | j | <q Wn | j j | i | _ d | _
d S( s Write the in memory keys down to disk to cap memory consumption.
If we already have some keys written to disk, we will combine them so
as to preserve the sorted order. The algorithm for combining uses
powers of two. So on the first spill, write all mem nodes into a
single index. On the second spill, combine the mem nodes with the nodes
on disk to create a 2x sized disk index and get rid of the first index.
On the third spill, create a single new disk index, which will contain
the mem nodes, and preserve the existing 2x sized index. On the fourth,
combine mem with the first and second indexes, creating a new one of
size 4x. On the fifth create a single new one, etc.
t .s <temp>N( t _combine_backing_indicest _spill_mem_keys_and_combinet! _spill_mem_keys_without_combiningt BTreeGraphIndexR t get_transport_from_patht _fileRC R6 t appendR t rangeR7 R8 ( R t new_backing_filet sizet backing_post new_backing( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyRD s
c C@ s | j | j d t S( Nt allow_optimize( t _write_nodest _iter_mem_nodesR9 ( R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyRM s c C@ s | j g } d } xJ t | j D]9 \ } } | d k rK | d 8} Pn | j | j q% W| d } | j | j | d t \ } } | | | f S( Nii RW ( RY t enumerateR6 R RQ t iter_all_entriesRX t _iter_smallestR9 ( R t iterators_to_combinet post backingRU RS RT ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyRL s
c C@ sd | j r9 xT | D]" \ } } } | j | | | q Wn' x$ | D] \ } } | j | | q@ Wd S( sn Add nodes to the index.
:param nodes: An iterable of (key, node_refs, value) entries to add.
N( R2 RI ( R R RE RF RH ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt add_nodes s
c c@ s | j } | j rM xm t | D]' } | | \ } } | | | | f Vq Wn5 x2 t | D]$ } | | \ } } | | | f VqZ Wd S( s& Iterate over the nodes held in memory.N( R7 R2 t sorted( R R RE RG RF ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyRY s c
c@ s t | d k r0 x | d D] } | Vq Wd Sg } xF | D]> } y | j | j Wq= t k
rz | j d q= Xq= Wd } x t rg t | D]* } | d d k r | d d | f ^ q } t | s d St | } | d } | | d d k rt j | | n | d d } | f | d d V| d } y | | j | | <Wq t k
r{d | | <q Xq Wd S( Ni i (
RC RQ t nextt
StopIterationR R. RZ t minR R@ (
R R] RF t current_valuest iteratort lastt itemt
candidatest selectedR^ ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR\ s6
:
c C@ s t } | d j d k r<t } x t | d D] \ } } | j d k r0 t } | j d k rm | t 8} n | r | j } n t } t j
| d d | | _ | j j t | j j t
t | | d j d q0 q0 Wt } | d j d k r| t 8} n t j
| d | j | d _ | d j j t n | d j j | r| rjt j | n | d j | d }
t } x> t | d D], } | j j |
r| j qt } PqW| r{d t j k rt j d n t } d }
| j d | t j
t t |
d | j | _ | j j t | j j t
t | d j d d | j j |
n | j | | | d | n d S(
sD Add a key to the current chunk.
:param string_key: The key to add.
:param line: The fully serialised key and value.
:param allow_optimize: If set to False, prevent setting the optimize
flag when writing out. This is used by the _spill_mem_keys_to_disk
functionality.
ii t optimize_for_sizei s
R s Inserting new global row.RW N( R9 R R R. RZ R" R R R: R t ChunkWriterR t _INTERNAL_FLAGt _INTERNAL_OFFSETt strt
_LEAF_FLAGR t BadIndexKeyR* t reversedR t debug_flagsR
t mutterR/ t insertt _add_key( R t
string_keyt linet rowsRW t new_leafR^ t internal_rowt lengthRk t key_linet new_rowt rowt reserved_bytes( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyRv " s`
'
c C@ s g } d } g | _ xk | D]c } | d k rA | j t n | d 7} t j | | j \ } } | j | | | d | q Wx6 t | D]( } t | t k } | j d | q Wt
g }
|
j t t | j d |
j t
t | j d |
j t t | d g | D] } | j ^ q#} |
j t d j t t | d | r| d d k rt j d d } n t j } | j |
t t t |
}
t } |
t k rt d
|
t f n x | D] } t } | j j | j j d | j j! t" } | j# | | t | t" k r`| j# d | |
n d }
t$ j% | j | } | | j d t" k rt | t k rt d | j d t" | f qqqW| j | j& } | j d | | f S(
s Write node_iterator out as a B+Tree.
:param node_iterator: An iterator of sorted nodes. Each node should
match the output given by iter_all_entries.
:param allow_optimize: If set to False, prevent setting the optimize
flag when writing out. This is used by the _spill_mem_keys_to_disk
functionality.
:return: A file handle for a temporary file containing a B+Tree for
the nodes.
i i RW R$ s
t ,iR s
bzr-index-s7 Could not fit the header in the reserved space: %d > %dR s5 Incorrect amount of data copied expected: %d, got: %d(' t row_lengthsRQ R0 t _btree_serializert
_flatten_nodeR2 Rv Rr t typeR* t _BTSIGNATURER Ro R
RA R R t _OPTION_ROW_LENGTHSt joint mapR t NamedTemporaryFileR R R t sumRC R. R R# R t flusht seekt readR" R R t pumpfileR! ( R t
node_iteratorRW Ry t key_countt nodeRw Rx R R$ t linesR t resultt positiont root_rowt reservedt
copied_lenRT ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyRX n s^
'
c C@ s | j | j d S( s Finalise the index.
:return: A file handle for a temporary file containing the nodes added
to the index.
i ( RX R[ ( R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s c C@ s d t j k r" t j d d n | j g } x3 | j D]( } | d k r; | j | j q; q; Wt | d k r | d S| j
| S( s Iterate over all keys within the index
:return: An iterable of (index, key, value, reference_lists). There is
no defined order for the result iteration - it will be in the most
efficient order for the index (in this case dictionary hash order).
t evili s- iter_all_entries scales with size of history.i i N( R Rs R
t mutter_callsiteRY R6 R RQ R[ RC R\ ( R t iteratorsR_ ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR[ s
c c@ s0 t | } | j } g | D] } | | k r | ^ q } | j rz xa | D]) } | | } | | | d | d f VqJ Wn- x* | D]" } | | } | | | d f Vq W| j s d S| j | xh | j D]] } | d k r q n | s d Sx8 | j | D]' } | j | d | f | d Vq Wq Wd S( s_ Iterate over keys within the index.
:param keys: An iterable providing the keys to be retrieved.
:return: An iterable of (index, key, value, reference_lists). There is no
defined order for the result iteration - it will be in the most
efficient order for the index (keys iteration order in this case).
i i N( t setR7 R2 R6 t difference_updateR t iter_entriest remove( R t keysR RE t
local_keysR R_ ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s* %
c
c@ s t | } | s d SxM | j D]B } | d k r8 q n x' | j | D] } | f | d VqH Wq W| j d k r2x | D] } | d d k r t j | n t | | j k r t j | n y | j | } Wn t k
r q| n X| j
r| | | d | d f Vq| | | | d f Vq| Wd Sxq| D]i} | d d k rat j | n t | | j k rt j | n | j } t | } yB x; t | r| d d k r| | d } | j
d qWWn t k
rq9n Xt | r| g } x | r| j
d } | j j \ } } t | t k rf| j | j qx& | j D] } | f t | VqsWqWq9| f | Vq9Wd S( sl Iterate over keys within the index using prefix matching.
Prefix matching is applied within the tuple of a key, not to within
the bytestring of each key element. e.g. if you have the keys ('foo',
'bar'), ('foobar', 'gam') and do a prefix search for ('foo', None) then
only the former key is returned.
:param keys: An iterable providing the key prefixes to be retrieved.
Each key prefix takes the form of a tuple the length of a key, but
with the last N elements 'None' rather than a regular bytestring.
The first element cannot be 'None'.
:return: An iterable as per iter_all_entries, but restricted to the
keys with a matching prefix to those supplied. No additional keys
will be returned, and every match that is in the index will be
returned.
Ni i i( R R6 R t iter_entries_prefixRA R Rq RC R7 t KeyErrorR2 t _get_nodes_by_keyt listt popt iteritemsRb R t dictt extendt
itervaluest tuple(
R R R_ R RE t key_dictt elementst dictsRh RF ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR sZ
c C@ s | j d k r i } | j r x | j j D]V \ } \ } } | } x$ | d D] } | j | i } qQ W| | | f | | d <q. Wng xd | j j D]S \ } \ } } | } x$ | d D] } | j | i } q W| | f | | d <q W| | _ n | j S( Ni( R8 R R2 R7 R t
setdefault( R t nodes_by_keyRE RG RF R t subkey( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR L s " " c C@ s$ t | j t d | j D S( sw Return an estimate of the number of keys in this index.
For InMemoryGraphIndex the estimate is exact.
c s@ s' | ] } | d k r | j Vq d S( N( R R ( t .0R_ ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pys <genexpr>c s ( RC R7 R R6 ( R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR ^ s c C@ s d S( s9 In memory index's have no known corruption at the moment.N( ( R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt validatef s ( ( R+ R, R- R RI RD RM RL R` RY R\ R. Rv RX R R[ R R R R R ( ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR1 o s$ # L M ) L t _LeafNodec B@ s/ e Z d Z d Z d Z d Z d Z RS( s* A leaf node for a serialised B+Tree index.t min_keyt max_keyt _keysc C@ sy t j | | | } | r@ | d d | _ | d d | _ n d | _ | _ t t | j | t | | _ d S( s) Parse bytes to create a leaf node object.i iN(
R t _parse_leaf_linesR R R t superR R R R ( R t bytest
key_lengtht ref_list_lengtht key_list( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR o s c C@ s | j } | j | S( s2 Return a sorted list of (key, (value, refs)) items( t itemst sort( R R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt all_items| s
c C@ s | j } | j | S( s! Return a sorted list of all keys.( R R ( R R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt all_keys s
( s min_keys max_keys _keys( R+ R, R- t __slots__R R R ( ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR j s
t
_InternalNodec B@ s& e Z d Z d Z d Z d Z RS( s/ An internal node for a serialised B+Tree index.R t offsetc C@ s | j | j d | _ d S( s. Parse bytes to create an internal node object.s
N( t _parse_linest splitR ( R R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s c C@ s} g } t | d d | _ t j j } xM | d D]A } | d k rJ Pn | j | t t | j d j q4 W| S( Ni i i t R ( t intR R R< R= RQ R R> R ( R R R t as_stRx ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s /( s keyss offset( R+ R, R- R R R ( ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s RN c B@ s e Z d Z e d d Z d Z d Z d Z d Z d Z d Z
d Z d
Z d Z
d Z d
Z d Z d Z d Z d Z d Z d Z e d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z d Z! RS( s Access to nodes via the standard GraphIndex interface for B+Tree's.
Individual nodes are held in a LRU cache. This holds the root node in
memory except when very large walks are done.
i c C@ s | | _ | | _ | | _ d | _ | j | _ d | _ | | _ t | _
d | _ | rr i | _ i | _
n$ t j t | _ t j d | _
d | _ d | _ d | _ d S( s^ Create a B+Tree index object on the index name.
:param transport: The transport to read data for the index from.
:param name: The file name of the index on transport.
:param size: Optional size of the index in bytes. This allows
compatibility with the GraphIndex API, as well as ensuring that
the initial read (to read the root node header) can be done
without over-reading even on empty indices, and on small indices
allows single-IO to read the entire index.
:param unlimited_cache: If set to True, then instead of using an
LRUCache with size _NODE_CACHE_SIZE, we will use a dict and always
cache all leaf nodes.
:param offset: The start of the btree index data isn't byte 0 of the
file. Instead it starts at some point later.
id N( t
_transportt _namet _sizeR RP t _compute_recommended_pagest _recommended_pagest
_root_nodet _base_offsetR t
_leaf_factoryt _leaf_value_cachet _leaf_node_cachet _internal_node_cacheR t LRUCachet _NODE_CACHE_SIZER t FIFOCachet
_key_countt _row_lengthst _row_offsets( R R t nameRT t unlimited_cacheR ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s" c C@ sL t | t | k oK | j | j k oK | j | j k oK | j | j k S( s@ Equal when self and other were created with the same parameters.( R R R R ( R t other( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt __eq__ s c C@ s | j | S( N( R ( R R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt __ne__ s c C@ s i } d } x | j t | D]s \ } } | d k rF | | _ nE | d k rb | j d } n | | k r~ | | j | <n
| | j | <| | | <q" W| S( s Read nodes and cache them in the lru.
The nodes list supplied is sorted and then read from disk, each node
being inserted it into the _node_cache.
Note: Asking for more nodes than the _node_cache can contain will
result in some of the results being immediately discarded, to prevent
this an assertion is raised if more nodes are asked for than are
cachable.
:return: A dict of {node_pos: node}
i iN( R t _read_nodesRa R R R R ( R R t foundt start_of_leavest node_posR ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt _get_and_cache_nodes s
"
c C@ s2 | j j } t t j | t t } | S( s Convert transport's recommended_page_size into btree pages.
recommended_page_size is in bytes, we want to know how many _PAGE_SIZE
pages fit in that length.
( R t recommended_page_sizeR t matht ceilt floatR" ( R t recommended_readt recommended_pages( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s c C@ s^ | j d k r t d n | j d k r8 | j d St t j | j t t } | S( s How many pages are in the index.
If we have read the header we will use the value stored there.
Otherwise it will be computed based on the length of the index.
sJ _compute_total_pages_in_index should not be called when self._size is NoneiN(
R R R# R R R R R R R" ( R t total_pages( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt _compute_total_pages_in_index s "c C@ s d t j k r( t j d | j | n t | | j k ro d t j k rk t j d t | | j n | S| j d k r d t j k r t j d n | S| j } | j
} | t | | j k r8| rg t | D] } | | k r | ^ q } n t | } d t j k r4t j d | n | S| j
d k rP| } nk t | j } t | | k rt | d k rd t j k rt j d n | S| j | | | } t | } d t j k rt j d | n | S(
s Find extra pages to download.
The idea is that we always want to make big-enough requests (like 64kB
for http), so that we don't waste round trips. So given the entries
that we already have cached and the new pages being downloaded figure
out what other pages we might want to read.
See also doc/developers/btree_index_prefetch.txt for more details.
:param offsets: The offsets to be read
:return: A list of offsets to download
R s expanding: %s offsets: %ss( not expanding large request (%s >= %s)s* not expanding without knowing index sizes reading all unread pages: %si s not expanding on first readss
expanded: %sN( R Rs R
Rt R RC R R R R t _get_offsets_to_cached_pagest xrangeRR R R t _expand_to_neighborsRa ( R t offsetsR t cached_offsetst xt expandedt
final_offsetst
tree_depth( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt _expand_offsets sD
$ c C@ s- t | } d } } t | } xt | | j k r(| r(t } x | D] } | d k rz | j | \ } } n | d }
|
d k r |
| k r |
| k r |
| k r | j |
n | d } | | k rP | | k rP | | k rP | | k rP | j | qP qP W| j | | } q% W| S( sM Expand requests to neighbors until we have enough pages.
This is called from _expand_offsets after policy has determined that we
want to expand.
We only want to expand requests within a given layer. We cheat a little
bit and assume all requests will be in the same layer. This is true
given the current design, but if it changes this algorithm may perform
oddly.
:param offsets: requested offsets
:param cached_offsets: offsets for pages we currently have cached
:return: A set() of offsets after expansion
i i N( R R RC R t _find_layer_first_and_endt addt update( R R R R R t firstt endt new_tipst next_tipsR^ t previoust after( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR U s.
c C@ s | j j d S( s Clear out any cached/memoized values.
This can be called at any time, but generally it is used when we have
extracted some information, but don't expect to be requesting any more
from this index.
N( R t clear( R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt clear_cache s c C@ s | j d k r | j n | d | j k rK t d | | j f n t } t } x: | j D], } | j | d | j | d | qj W| | S( Ni s& No ref list %d, index has %d ref listsi ( R R t _get_root_nodet node_ref_listst
ValueErrorR R[ R R ( R t ref_list_numR t refsR ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt external_references s
c C@ sD d } } x- | j D]" } | } | } | | k r Pq q W| | f S( s Find the start/stop nodes for the layer corresponding to offset.
:return: (first, end)
first is the first node in this layer
end is the first node of the next layer
i ( R ( R R R R t roffset( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s
c C@ sN t | j j } | j | j j | j d k rJ | j d n | S( s, Determine what nodes we already have cached.i N( R R R R R R R R ( R R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s
c C@ s) | j d k r" | j d g n | j S( Ni ( R R t _get_internal_nodes( R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s c C@ s i } g } xo | D]g } | d k rG | j d k rG | j | d <q n y | | | | <Wq t k
ry | j | q Xq W| s | S| j | } | j | j | | S( Ni ( R R R RQ R R R ( R t cachet node_indexesR t neededt idx( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt
_get_nodes s
c C@ s | j | j | S( s\ Get a node, from cache or disk.
After getting it, the node will be cached.
( R R ( R R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s c C@ sj | j d k rf xT | j D]C } x: | j D], \ } } | | j k rN Pn | | j | <q/ Wq Wn d S( s5 Cache directly from key => value, skipping the btree.N( R R R R ( R R R RE RF ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt _cache_leaf_values s c C@ s& | j | j | } | j | | S( s) Get a bunch of nodes, from cache or disk.( R R R ( R R R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt _get_leaf_nodes s
c
c@ s d t j k r" t j d d n | j s2 d S| j d d k r | j r xk | j j D]# \ } \ } } | | | | f Vq^ Wn4 x1 | j j D] \ } \ } } | | | f Vq Wd S| j d } | j d } t | | } | d g k r
d | j f g } n | j
| } | j rsx | D]@ \ } } x1 | j D]# \ } \ } } | | | | f VqEWq,WnH xE | D]= \ } } x. | j D] \ } \ } } | | | f VqWqzWd S( s Iterate over all keys within the index.
:return: An iterable of (index, key, value) or (index, key, value, reference_lists).
The former tuple is used when there are no reference lists in the
index, making the API compatible with simple key:value index types.
There is no defined order for the result iteration - it will be in
the most efficient order for the index.
R i s- iter_all_entries scales with size of history.Nii ii ( R Rs R
R R R R R R RR R (
R RE RF R R t
end_of_leavest needed_offsetsR R& R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR[ s2
""
c C@ s | s
g S| s d | f g St | d k rL t j | | d | f g St | } t | } | j } | j \ } } d t f d Y} d t f d Y} g } g }
y x t r| | k r>g } | | f }
| j |
xJ | | k r:| j | y | j } Wq t k
r6| q Xq Wn x@ | | k ry | j \ } } WqAt k
r|| qAXqAWq WWnV | k
rnF | k
r| g } | j
| t | | f }
| j |
n X| S( so Find the positions where each 'in_key' would fit in fixed_keys.
This is equivalent to doing "bisect_right" on each in_key into
fixed_keys
:param in_keys: A sorted list of keys to match with fixed_keys
:param fixed_keys: A sorted list of keys to match against
:return: A list of (integer position, [key list]) tuples.
i i t InputDonec B@ s e Z RS( ( R+ R, ( ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR 3 s t FixedDonec B@ s e Z RS( ( R+ R, ( ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR 4 s ( RC t bisectt bisect_rightt iterRZ Rb t ExceptionR. RQ Rc R ( t in_keyst
fixed_keyst in_keys_itert fixed_keys_itert
cur_in_keyt cur_fixed_offsett
cur_fixed_keyR R t outputt cur_outt cur_keys( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt _multi_bisect_right sN
c C@ s! d t | f g } x t | j d d ! D] \ } } g | D] \ } } | ^ q? } | j | } g } xp | D]h \ }
} | |
} | j | | j }
| | j } | j g |
D] \ } } | | | f ^ q qs W| } q, Wg | D] \ } } | ^ q } | j | } | | f S( s6 Take the given set of keys, and find the corresponding LeafNodes.
:param keys: An unsorted iterable of keys to search for
:return: (nodes, index_and_keys)
nodes is a dict mapping {index: LeafNode}
keys_at_index is a list of tuples of [(index, [keys for Leaf])]
i i i( Ra RZ R R R# R R R R ( R R t
keys_at_indext row_post next_row_startR
t s_keysR R t next_nodes_and_keyst
node_indext sub_keysR t positionst node_offsetR^ ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt _walk_through_internal_nodes^ s #
+
c
c@ st t | } | s d S| j s& d Sg } | j d k rD | } ny xv | D]n } | j j | d } | d k r | \ } } | j r | | | | f Vq | | | f VqK | j | qK Wd } | } | s d S| j | \ } } x | D]} \ } }
|
sq n | | } xX |
D]P } | | k r| | \ } } | j rW| | | | f Vqh| | | f VqqWq Wd S( sH Iterate over keys within the index.
:param keys: An iterable providing the keys to be retrieved.
:return: An iterable as per iter_all_entries, but restricted to the
keys supplied. No additional keys will be returned, and every
key supplied that is in the index will be returned.
N( t frozensetR R R t getR RQ R- (
R R t needed_keysRE RF R t last_keyR t nodes_and_keysR) R* R t next_sub_key( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s>
c C@ s | j s | j | t S| | j k rK t d | | j f n | j | \ } } t } xa| D]Y\ } } | s qp n | | }
t } x[ | D]S } | |
k r | j | q |
| \ }
} | | } | | | <| j | q W| j | } x | rt } x | D] } | |
k rg|
| \ }
} | | } | | | <| j | q!| |
j k r| j | q!| |
j k r| j | q!| j | q!W| j | } q Wqp W| j | j | } | S( s8 Find the parent_map information for the set of keys.
This populates the parent_map dict and missing_keys set based on the
queried keys. It also can fill out an arbitrary number of parents that
it finds while searching for the supplied keys.
It is unlikely that you want to call this directly. See
"CombinedGraphIndex.find_ancestry()" for a more appropriate API.
:param keys: A keys whose ancestry we want to return
Every key will either end up in 'parent_map' or 'missing_keys'.
:param ref_list_num: This index in the ref_lists is the parents we
care about.
:param parent_map: {key: parent_keys} for keys that are present in this
index. This may contain more entries than were in 'keys', that are
reachable ancestors of the keys requested.
:param missing_keys: keys which are known to be missing in this index.
This may include parents that were not directly requested, but we
were able to determine that they are not present in this index.
:return: search_keys parents that were found but not queried to know
if they are missing or present. Callers can re-query this index for
those keys, and they will be placed into parent_map or missing_keys
s& No ref list %d, index has %d ref lists(
R R R R R R- R t
differenceR R ( R R R t
parent_mapt missing_keysR R2 t parents_not_on_pageR) R* R t parents_to_checkR3 RF R t parent_keyst next_parents_to_checkRE t search_keys( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt _find_ancestors sL
c c@ s t t | } | s d S| j d k r8 | j n i } | j r| j d k r x | j D]" \ } } } } | | f | | <qc Wqi } x| j D]\ \ } } } } | | | f } | } x$ | d D] }
| j |
i } q W| | | d <q Wn | j d k r>x | j D] \ } } } | | | <q Wnm i } xd | j D]V \ } } } | | f } | } x$ | d D] }
| j |
i } q}W| | | d <qQW| j d k rox | D] } | d d k rt j
| n t | | j k rt j
| n yC | j r@| | \ } } | | | | f Vn | | | | f VWqt k
rfqXqWd Sxe| D]]} | d d k rt j
| n t | | j k rt j
| n | } t
| } yB x; t | r| d d k r| | d } | j d qWWn t k
r/qvn Xt | r| g }
x |
r|
j d } | j j \ } } t | t k r|
j | j qHx | j D] } | f | VqWqHWqv| f | VqvWd S( s Iterate over keys within the index using prefix matching.
Prefix matching is applied within the tuple of a key, not to within
the bytestring of each key element. e.g. if you have the keys ('foo',
'bar'), ('foobar', 'gam') and do a prefix search for ('foo', None) then
only the former key is returned.
WARNING: Note that this method currently causes a full index parse
unconditionally (which is reasonably appropriate as it is a means for
thunking many small indices into one larger one and still supplies
iter_all_entries at the thunk layer).
:param keys: An iterable providing the key prefixes to be retrieved.
Each key prefix takes the form of a tuple the length of a key, but
with the last N elements 'None' rather than a regular bytestring.
The first element cannot be 'None'.
:return: An iterable as per iter_all_entries, but restricted to the
keys with a matching prefix to those supplied. No additional keys
will be returned, and every match that is in the index will be
returned.
Ni ii ( Ra R R R R R RA R[ R R Rq RC R R R R Rb R R R R ( R R R t _1RE RF R R t key_valueR R RH R R Rh ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR 5 s
c C@ s# | j d k r | j n | j S( s Return an estimate of the number of keys in this index.
For BTreeGraphIndex the estimate is exact as it is contained in the
header.
N( R R R ( R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR s
c C@ sQ g } d } x( | j D] } | j | | | 7} q W| j | | | _ d S( s: Fill out the _row_offsets attribute based on _row_lengths.i N( R RQ R ( R R t
row_offsetR ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt _compute_row_offsets s
c C@ sx | d t | j !} | | j k sC t j | j t n | t | j j } | d } | j t s t j | n y t
| t t | _ Wn t k
r t j | n X| d } | j t
s t j | n y t
| t t
| _ Wn t k
r4t j | n X| d } | j t s`t j | n y t
| t t | _ Wn t k
rt j | n X| d } | j t st j | n yH t t
g | t t j d D] } t | r| ^ q | _ Wn t k
r5t j | n X| j t | t t t | d d ! d } | | | f S( s Parse the header from a region of bytes.
:param bytes: The data to parse.
:return: An offset, data tuple such as readv yields, for the unparsed
data. (which may be of length 0).
i i i i R i ( RC t
_signatureR t BadIndexFormatSignatureR RN t
splitlinest
startswithR t BadIndexOptionsR R R R
RA R R R R R R R@ R ( R R t signatureR t options_lineR| t
header_end( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt _parse_header_from_bytes sJ
"
*c
c@ s d } g } | j } x
| D]} | t } t } | d k r | j r\ t t | j } q| j j | j } t | } | | | _ g t | | t D] } | t t | | f ^ q } PnA | | j k r t
d | | j f n t | | j | } | j | | | f q W| s,d S| d k rkg | D]# \ } } | | | | | !f ^ q?}
nv | j d k r| j j
| j | }
nL g }
xC | D]; \ } } | j j | |
j | | j j | f qWx |
D] \ } } | | 8} | d k r:| j | \ } } t | d k r:qq:n t j | } | j t rv| j | | j | j } n. | j t rt | } n t
d | | t | f VqWd S( s Read some nodes from disk into the LRU cache.
This performs a readv to get the node data into memory, and parses each
node, then yields it to the caller. The nodes are requested in the
supplied order. If possible doing sort() on the list before requesting
a read may improve performance.
:param nodes: The nodes to read. 0 - first node, 1 - second node etc.
:return: None
i s. tried to read past the end of the file %s > %sNs Unknown node type for %r( R R R" R Rd R t get_bytesR RC R R# RQ RP t readvR R RI t zlibt
decompressRD Rp R RA R Rm R (
R R R t rangest base_offsetR R RT t num_bytest startt data_rangest dataR ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR sZ
5 0#
c C@ s t S( s' The file signature for this index type.( R ( R ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyRA / s c C@ si | j t | j d k r/ | j d } n d } | j d } x | j t | | D] } q[ Wd S( s6 Validate that everything in the index can be accessed.i iN( R RC R R R RR ( R t
start_nodet node_endR ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyR 3 s
(" R+ R, R- R9 R R R R R R R R R R R R R R R R R R[ t staticmethodR# R- R R< R R R@ RI R RA R ( ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyRN s> ' E ,
+R ! 8 ~ p
/ G ( t _btree_serializer_pyx( t _btree_serializer_py(0 R- t
__future__R R t bzrlib.lazy_importR t globalst bzrlibR R R R R R R R R
R t bzrlib.indexR R
R R R Rp Rm Rn R R" R t objectR R/ R0 R4 R1 R R R RN t _gcchk_factoryRW R t _parse_into_chkt ImportErrort et failed_to_load_extensionRX ( ( ( s6 /usr/lib/python2.7/dist-packages/bzrlib/btree_index.pyt <module> sB F &
Copyright © 2017 || Recoded By Mr.Bumblebee