o hk@s.ddlmZddlZddlZddlZddlZddlmZmZm Z m Z m Z ddl m Z mZmZmZmZddlmZmZddlmZdZz ddlmZd ZWn eyUYnwd ZGd d d eZGd ddeZGdddZGdddeZ GdddeZ!GdddeZ"GdddZ#GdddZ$dS))BytesION)msb_size stream_copyapply_delta_dataconnect_deltas delta_types)allocate_memory LazyMixinmake_shawriteclose) NULL_BYTE BYTE_SPACE) force_bytesF) apply_deltaT) DecompressMemMapReaderFDCompressedSha1WriterDeltaApplyReader Sha1WriterFlexibleSha1WriterZippedStoreShaWriterrFDStream NullStreamc@s~eZdZdZdZdZdddZddZd d Zd d Z e dddZ ddZ ddZ ddZeeddfddZdddZdS) raReads data in chunks from a memory map and decompresses it. The client sees only the uncompressed data, respective file-like read calls are handling on-demand buffered decompression accordingly A constraint on the total size of bytes is activated, simulating a logical file within a possibly larger physical memory area To read efficiently, you clearly don't want to read individual bytes, instead, read a few kilobytes at least. **Note:** The chunk-size should be carefully selected as it will involve quite a bit of string copying due to the way the zlib is implemented. Its very wasteful, hence we try to find a good tradeoff between allocation time and number of times we actually allocate. An own zlib implementation would be good here to better support streamed reading - it would only need to keep the mmap and decompress it into chunks, that's all ... ) _m_zip_buf_buflen_br_cws_cwe_s_close_cbr_phiiNcCsR||_t|_d|_d|_|dur||_d|_d|_d|_ d|_ d|_ ||_ dS)z|Initialize with mmap for stream reading :param m: must be content data - use new if you have object data and no sizeNrF) rzlib decompressobjrrrr rrrr"r#r!)selfmclose_on_deletionsizer*@/var/www/vscode/kcb/lib/python3.10/site-packages/gitdb/stream.py__init__Es  zDecompressMemMapReader.__init__cCs|dksJ|dS)Nr )_parse_header_infor&attrr*r*r+ _set_cache_Us  z"DecompressMemMapReader._set_cache_cCs |dSN)r r&r*r*r+__del__[ zDecompressMemMapReader.__del__cCs~d}||_||}|t}|d|t\}}t|}||_d|_|d7}t||d|_ t |||_ d|_ ||fS)zIf this stream contains object data, parse the header info and skip the stream to a point where each read will yield object content :return: parsed type_string, sizei NrT) r readfindr splitrintrrrlenrr#)r&maxbhdrhdrendtypr)r*r*r+r-^s  z)DecompressMemMapReader._parse_header_infoFcCs"t||d}|\}}|||fS)aCreate a new DecompressMemMapReader instance for acting as a read-only stream This method parses the object header from m and returns the parsed type and size, as well as the created stream instance. :param m: memory map on which to operate. It must be object data ( header + contents ) :param close_on_deletion: if True, the memory map will be closed once we are being deletedr)rr-)r&r'r(instr>r)r*r*r+new{s  zDecompressMemMapReader.newcC|jS)z8:return: random access compatible data we are working on)rr2r*r*r+dataszDecompressMemMapReader.datacCs*|jrt|jdr|jd|_dSdS)zClose our underlying stream of compressed bytes if this was allowed during initialization :return: True if we closed the underlying stream :note: can be called safely r FN)r!hasattrrr r2r*r*r+r s    zDecompressMemMapReader.closecCs|j|jkrJ|jjsJd|_t|jdr(|jjtjkr'|t j |jjtjksn|jjsF|j t |j krF|t j |jjsF|j t |j ks4|j|_|j S)z :return: number of compressed bytes read. This includes the bytes it took to decompress the header ( if there was one )rstatus)rr r unused_datarCrDr$Z_OKr6mmapPAGESIZEr"r:rr2r*r*r+compressed_bytes_reads   z,DecompressMemMapReader.compressed_bytes_readSEEK_SETrcCsZ|dks |ttddkrtdt|_d|_|_|_|_ |j r+d|_ |` dSdS)zgAllows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0rrJCan only seek to position 0FN) getattros ValueErrorr$r%rrrrr"r#r r&offsetwhencer*r*r+seeks zDecompressMemMapReader.seekcCs|dkr |j|j}n t||j|j}|dkrdSd}|jrR|j|kr:|j|}|j|8_|j|7_|S|j}||j8}|j|j7_d|_d|_|jj}|rg|jt ||_ |j ||_n |j }|j|_ |||_|j|j dkr|j d|_|j |j |j}|j t ||_|j ||}t tdtjdvrtjdkst |jj}n t |jjt |jj}|jt ||7_|jt |7_|r||}|rt |t ||kr|j|jkr|||t |7}|S)Nr5rZLIB_RUNTIME_VERSION)z1.2.7z1.2.5darwin)r rminrrr6runconsumed_tailrr:rr decompressrLr$ ZLIB_VERSIONsysplatformrEr")r&r)dattailcwsindatadcompdatunused_datalenr*r*r+r6sN      $ zDecompressMemMapReader.readr1F)rS)__name__ __module__ __qualname____doc__ __slots__ max_read_sizer,r0r3r- classmethodr@rBr rIrLrMrRr6r*r*r*r+r.s    1rc@seZdZdZdZdZddZddZdd Ze seZ neZ dd d Z e e d d fddZeddZeddZeddZeddZdS)raA reader which dynamically applies pack deltas to a base object, keeping the memory demands to a minimum. The size of the final object is only obtainable once all deltas have been applied, unless it is retrieved from a pack index. The uncompressed Delta has the following layout (MSB being a most significant bit encoded dynamic size): * MSB Source Size - the size of the base against which the delta was created * MSB Target Size - the size of the resulting data after the delta was applied * A list of one byte commands (cmd) which are followed by a specific protocol: * cmd & 0x80 - copy delta_data[offset:offset+size] * Followed by an encoded offset into the delta data * Followed by an encoded size of the chunk to copy * cmd & 0x7f - insert * insert cmd bytes from the delta buffer into the output stream * cmd == 0 - invalid operation ( or error in delta stream ) )_bstream _dstreams _mm_target_sizericCs:t|dks Jd|d|_t|dd|_d|_dS)zInitialize this instance with a list of streams, the first stream being the delta to apply on top of all following deltas, the last stream being the base object onto which to apply the deltasr5z+Need at least one delta and one base streamrSNr)r:rltuplermr)r& stream_listr*r*r+r,hs  zDeltaApplyReader.__init__cCst|jdkr ||St|j}|dkr!d|_td|_dS||_t|j|_t|jj }t |jj |j |jj dt j|jj }||||jddS)Nr5r)r:rm_set_cache_brute_rrboundrorrnrlr)rr6r rGrHapplyrR)r&r/dclbbufr r*r*r+_set_cache_too_slow_without_crs        z.DeltaApplyReader._set_cache_too_slow_without_ccCslt}d}|jD]&}|d}t|\}}t||\}}|||d|||ft||}q|jj} |}t|jdkrCt| |} }t | } t |jj| j | dt j t |} d} tt|t|jD]I\\} }}}}t |j|}| | t |j|j |jdt j dtvrt| || n t| ||t|| j | | } } | d| d|} qd| |_| |_dS)z*If we are here, we apply the actual deltasriNr5rr c_apply_delta)listrmr6rappendmaxrlr)r:rrr rGrHzipreversedglobalsryrrRrnro)r&r/buffer_info_listmax_target_sizedstreambufrPsrc_size target_size base_sizerwtbuffinal_target_sizedbufddatar*r*r+rss:    $      z"DeltaApplyReader._set_cache_brute_rcCsB|j|j}|dks||kr|}|j|}|jt|7_|S)Nr5)rorrnr6r:)r&countblrBr*r*r+r6s  zDeltaApplyReader.readrJcCs6|dks |ttddkrtdd|_|jddS)zhAllows to reset the stream to restart reading :raise ValueError: If offset and whence are not 0rrJrKN)rLrMrNrrnrRrOr*r*r+rRszDeltaApplyReader.seekcCs<t|dkr td|djtvrtd|dj||S)a Convert the given list of streams into a stream which resolves deltas when reading from it. :param stream_list: two or more stream objects, first stream is a Delta to the object that you want to resolve, followed by N additional delta streams. The list's last stream must be a non-delta stream. :return: Non-Delta OPackStream object whose stream can be used to obtain the decompressed resolved data :raise ValueError: if the stream list cannot be handledzNeed at least two streamsrSzNCannot resolve deltas if there is no base object stream, last one was type: %s)r:rNtype_idrtype)clsrqr*r*r+r@s  zDeltaApplyReader.newcC|jjSr1)rlrr2r*r*r+rzDeltaApplyReader.typecCrr1)rlrr2r*r*r+rrzDeltaApplyReader.type_idcCrA)z3:return: number of uncompressed bytes in the stream)ror2r*r*r+r)szDeltaApplyReader.sizeNr)rerfrgrhrik_max_memory_mover,rxrs has_perf_modr0r6rLrMrRrkr@propertyrrr)r*r*r*r+rBs(  "K     rc@s.eZdZdZdZddZddZd dd Zd S) rzpSimple stream writer which produces a sha whenever you like as it degests everything it is supposed to writesha1cCs t|_dSr1)r rr2r*r*r+r,2r4zSha1Writer.__init__cCs|j|t|S)z{:raise IOError: If not all bytes could be written :param data: byte object :return: length of incoming data)rupdater:r&rBr*r*r+r 7s zSha1Writer.writeFcCs|r|jS|jS)z]:return: sha so far :param as_hex: if True, sha will be hex-encoded, binary otherwise)r hexdigestdigest)r&as_hexr*r*r+shaDs  zSha1Writer.shaNrd)rerfrgrhrir,r rr*r*r*r+r,s  rc@s$eZdZdZdZddZddZdS)rzZWriter producing a sha1 while passing on the written bytes to the given write functionwritercCst|||_dSr1)rr,r)r&rr*r*r+r,Ts  zFlexibleSha1Writer.__init__cCst||||dSr1)rr rrr*r*r+r Xs zFlexibleSha1Writer.writeN)rerfrgrhrir,r r*r*r*r+rNs  rc@sPeZdZdZdZddZddZddZd d Ze e d d fd dZ ddZ dS)rz=Remembers everything someone writes to it and generates a sha)rr}cCs$t|t|_ttj|_dSr1)rr,rrr$ compressobj Z_BEST_SPEEDr}r2r*r*r+r,bs zZippedStoreShaWriter.__init__cCs t|j|Sr1)rLrr.r*r*r+ __getattr__gr4z ZippedStoreShaWriter.__getattr__cCs$t||}|j|j||Sr1)rr rr}compress)r&rBalenr*r*r+r js zZippedStoreShaWriter.writecCs|j|jdSr1)rr r}flushr2r*r*r+r pszZippedStoreShaWriter.closerJrcCs0|dks |ttddkrtd|jddS)z`Seeking currently only supports to rewind written data Multiple writes are not supportedrrJrKN)rLrMrNrrRrOr*r*r+rRsszZippedStoreShaWriter.seekcCs |jS)zA:return: string value from the current stream position to the end)rgetvaluer2r*r*r+r{s zZippedStoreShaWriter.getvalueN) rerfrgrhrir,rr r rLrMrRrr*r*r*r+r]s rcs<eZdZdZdZedZfddZddZdd Z Z S) rzDigests data written to it, making the sha available, then compress the data and write it to the file descriptor **Note:** operates on raw file descriptors **Note:** for this to work, you have to use the close-method of this instance)fdrr}z+Failed to write all bytes to filedescriptorcs"t||_ttj|_dSr1)superr,rr$rrr}r&r __class__r*r+r,s zFDCompressedSha1Writer.__init__cCs>|j||j|}t|j|}|t|kr|jt|S)zZ:raise IOError: If not all bytes could be written :return: length of incoming data)rrr}rr rr:exc)r&rBcdata bytes_writtenr*r*r+r s    zFDCompressedSha1Writer.writecCs.|j}t|j|t|kr|jt|jSr1)r}rr rr:rr )r& remainderr*r*r+r s  zFDCompressedSha1Writer.close) rerfrgrhriIOErrorrr,r r __classcell__r*r*rr+rs  rc@sFeZdZdZdZddZddZddd Zd d Zd d Z ddZ dS)rzA simple wrapper providing the most basic functions on a file descriptor with the fileobject interface. Cannot use os.fdopen as the resulting stream takes ownership_fd_poscCs||_d|_dSNrrrr*r*r+r,s zFDStream.__init__cCs$|jt|7_t|j|dSr1)rr:rMr rrr*r*r+r szFDStream.writercCs:|dkr tj|j}t|j|}|jt|7_|Sr)rMpathgetsize _filepathr6rrr:)r&rbytesr*r*r+r6s z FDStream.readcCrAr1)rr2r*r*r+filenozFDStream.filenocCrAr1)rr2r*r*r+tellrz FDStream.tellcCst|jdSr1)r rr2r*r*r+r szFDStream.closeNr) rerfrgrhrir,r r6rrr r*r*r*r+rs   rc@s0eZdZdZeZd ddZddZddZd S) rzVA stream that does nothing but providing a stream interface. Use it like /dev/nullrcCsdS)Nr*)r&r)r*r*r+r6zNullStream.readcCsdSr1r*r2r*r*r+r rzNullStream.closecCst|Sr1)r:rr*r*r+r szNullStream.writeNr) rerfrgrhrprir6r r r*r*r*r+rs   r)%iorrGrMr\r$ gitdb.funrrrrr gitdb.utilrr r r r gitdb.constr rgitdb.utils.encodingrrgitdb_speedups._perfrry ImportError__all__rrrrrrrrr*r*r*r+s8    k"#("