api.py 123 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443
  1. # -*- coding: utf-8 -*-
  2. # MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C)
  3. # 2015, 2016, 2017 MinIO, Inc.
  4. #
  5. # Licensed under the Apache License, Version 2.0 (the "License");
  6. # you may not use this file except in compliance with the License.
  7. # You may obtain a copy of the License at
  8. #
  9. # http://www.apache.org/licenses/LICENSE-2.0
  10. #
  11. # Unless required by applicable law or agreed to in writing, software
  12. # distributed under the License is distributed on an "AS IS" BASIS,
  13. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. # See the License for the specific language governing permissions and
  15. # limitations under the License.
  16. # pylint: disable=too-many-arguments
  17. # pylint: disable=too-many-branches
  18. # pylint: disable=too-many-function-args
  19. # pylint: disable=too-many-lines
  20. # pylint: disable=too-many-public-methods
  21. # pylint: disable=too-many-statements
  22. # pylint: disable=too-many-locals
  23. # pylint: disable=too-many-positional-arguments
  24. """
  25. Simple Storage Service (aka S3) client to perform bucket and object operations.
  26. """
  27. from __future__ import absolute_import, annotations
  28. import itertools
  29. import json
  30. import os
  31. import tarfile
  32. from collections.abc import Iterable
  33. from datetime import datetime, timedelta
  34. from io import BytesIO
  35. from random import random
  36. from typing import (Any, BinaryIO, Iterator, Optional, TextIO, Tuple, Union,
  37. cast)
  38. from urllib.parse import urlunsplit
  39. from xml.etree import ElementTree as ET
  40. import certifi
  41. import urllib3
  42. from urllib3 import Retry
  43. from urllib3._collections import HTTPHeaderDict
  44. try:
  45. from urllib3.response import BaseHTTPResponse # type: ignore[attr-defined]
  46. except ImportError:
  47. from urllib3.response import HTTPResponse as BaseHTTPResponse
  48. from urllib3.util import Timeout
  49. from . import time
  50. from .commonconfig import (COPY, REPLACE, ComposeSource, CopySource,
  51. SnowballObject, Tags)
  52. from .credentials import Credentials, StaticProvider
  53. from .credentials.providers import Provider
  54. from .datatypes import (Bucket, CompleteMultipartUploadResult, EventIterable,
  55. ListAllMyBucketsResult, ListMultipartUploadsResult,
  56. ListPartsResult, Object, Part, PostPolicy,
  57. parse_copy_object, parse_list_objects)
  58. from .deleteobjects import (DeleteError, DeleteObject, DeleteRequest,
  59. DeleteResult)
  60. from .error import InvalidResponseError, S3Error, ServerError
  61. from .helpers import (_DEFAULT_USER_AGENT, MAX_MULTIPART_COUNT,
  62. MAX_MULTIPART_OBJECT_SIZE, MAX_PART_SIZE, MIN_PART_SIZE,
  63. BaseURL, DictType, ObjectWriteResult, ProgressType,
  64. ThreadPool, check_bucket_name, check_object_name,
  65. check_sse, check_ssec, genheaders, get_part_info,
  66. headers_to_strings, is_valid_policy_type, makedirs,
  67. md5sum_hash, queryencode, read_part_data, sha256_hash)
  68. from .legalhold import LegalHold
  69. from .lifecycleconfig import LifecycleConfig
  70. from .notificationconfig import NotificationConfig
  71. from .objectlockconfig import ObjectLockConfig
  72. from .replicationconfig import ReplicationConfig
  73. from .retention import Retention
  74. from .select import SelectObjectReader, SelectRequest
  75. from .signer import presign_v4, sign_v4_s3
  76. from .sse import Sse, SseCustomerKey
  77. from .sseconfig import SSEConfig
  78. from .tagging import Tagging
  79. from .versioningconfig import VersioningConfig
  80. from .xml import Element, SubElement, findtext, getbytes, marshal, unmarshal
  81. class Minio:
  82. """
  83. Simple Storage Service (aka S3) client to perform bucket and object
  84. operations.
  85. :param endpoint: Hostname of a S3 service.
  86. :param access_key: Access key (aka user ID) of your account in S3 service.
  87. :param secret_key: Secret Key (aka password) of your account in S3 service.
  88. :param session_token: Session token of your account in S3 service.
  89. :param secure: Flag to indicate to use secure (TLS) connection to S3
  90. service or not.
  91. :param region: Region name of buckets in S3 service.
  92. :param http_client: Customized HTTP client.
  93. :param credentials: Credentials provider of your account in S3 service.
  94. :param cert_check: Flag to indicate to verify SSL certificate or not.
  95. :return: :class:`Minio <Minio>` object
  96. Example::
  97. # Create client with anonymous access.
  98. client = Minio("play.min.io")
  99. # Create client with access and secret key.
  100. client = Minio("s3.amazonaws.com", "ACCESS-KEY", "SECRET-KEY")
  101. # Create client with access key and secret key with specific region.
  102. client = Minio(
  103. "play.minio.io:9000",
  104. access_key="Q3AM3UQ867SPQQA43P2F",
  105. secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
  106. region="my-region",
  107. )
  108. **NOTE on concurrent usage:** `Minio` object is thread safe when using
  109. the Python `threading` library. Specifically, it is **NOT** safe to share
  110. it between multiple processes, for example when using
  111. `multiprocessing.Pool`. The solution is simply to create a new `Minio`
  112. object in each process, and not share it between processes.
  113. """
  114. _region_map: dict[str, str]
  115. _base_url: BaseURL
  116. _user_agent: str
  117. _trace_stream: Optional[TextIO]
  118. _provider: Optional[Provider]
  119. _http: urllib3.PoolManager
  120. def __init__(
  121. self,
  122. endpoint: str,
  123. access_key: Optional[str] = None,
  124. secret_key: Optional[str] = None,
  125. session_token: Optional[str] = None,
  126. secure: bool = True,
  127. region: Optional[str] = None,
  128. http_client: Optional[urllib3.PoolManager] = None,
  129. credentials: Optional[Provider] = None,
  130. cert_check: bool = True,
  131. ):
  132. # Validate http client has correct base class.
  133. if http_client and not isinstance(http_client, urllib3.PoolManager):
  134. raise ValueError(
  135. "HTTP client should be instance of `urllib3.PoolManager`"
  136. )
  137. self._region_map = {}
  138. self._base_url = BaseURL(
  139. ("https://" if secure else "http://") + endpoint,
  140. region,
  141. )
  142. self._user_agent = _DEFAULT_USER_AGENT
  143. self._trace_stream = None
  144. if access_key:
  145. if secret_key is None:
  146. raise ValueError("secret key must be provided with access key")
  147. credentials = StaticProvider(access_key, secret_key, session_token)
  148. self._provider = credentials
  149. # Load CA certificates from SSL_CERT_FILE file if set
  150. timeout = timedelta(minutes=5).seconds
  151. self._http = http_client or urllib3.PoolManager(
  152. timeout=Timeout(connect=timeout, read=timeout),
  153. maxsize=10,
  154. cert_reqs='CERT_REQUIRED' if cert_check else 'CERT_NONE',
  155. ca_certs=os.environ.get('SSL_CERT_FILE') or certifi.where(),
  156. retries=Retry(
  157. total=5,
  158. backoff_factor=0.2,
  159. status_forcelist=[500, 502, 503, 504]
  160. )
  161. )
  162. def __del__(self):
  163. if hasattr(self, "_http"): # Only required for unit test run
  164. self._http.clear()
  165. def _handle_redirect_response(
  166. self,
  167. method: str,
  168. response: BaseHTTPResponse,
  169. bucket_name: Optional[str] = None,
  170. retry: bool = False,
  171. ) -> tuple[Optional[str], Optional[str]]:
  172. """
  173. Handle redirect response indicates whether retry HEAD request
  174. on failure.
  175. """
  176. code, message = {
  177. 301: ("PermanentRedirect", "Moved Permanently"),
  178. 307: ("Redirect", "Temporary redirect"),
  179. 400: ("BadRequest", "Bad request"),
  180. }.get(response.status, (None, None))
  181. region = response.headers.get("x-amz-bucket-region")
  182. if message and region:
  183. message += "; use region " + region
  184. if (
  185. retry and region and method == "HEAD" and bucket_name and
  186. self._region_map.get(bucket_name)
  187. ):
  188. code, message = ("RetryHead", None)
  189. return code, message
  190. def _build_headers(
  191. self,
  192. host: str,
  193. headers: Optional[DictType] = None,
  194. body: Optional[bytes] = None,
  195. creds: Optional[Credentials] = None,
  196. ) -> tuple[DictType, datetime]:
  197. """Build headers with given parameters."""
  198. headers = headers or {}
  199. md5sum_added = headers.get("Content-MD5")
  200. headers["Host"] = host
  201. headers["User-Agent"] = self._user_agent
  202. sha256 = None
  203. md5sum = None
  204. if body:
  205. headers["Content-Length"] = str(len(body))
  206. if creds:
  207. if self._base_url.is_https:
  208. sha256 = "UNSIGNED-PAYLOAD"
  209. md5sum = None if md5sum_added else md5sum_hash(body)
  210. else:
  211. sha256 = sha256_hash(body)
  212. else:
  213. md5sum = None if md5sum_added else md5sum_hash(body)
  214. if md5sum:
  215. headers["Content-MD5"] = md5sum
  216. if sha256:
  217. headers["x-amz-content-sha256"] = sha256
  218. if creds and creds.session_token:
  219. headers["X-Amz-Security-Token"] = creds.session_token
  220. date = time.utcnow()
  221. headers["x-amz-date"] = time.to_amz_date(date)
  222. return headers, date
  223. def _url_open(
  224. self,
  225. method: str,
  226. region: str,
  227. bucket_name: Optional[str] = None,
  228. object_name: Optional[str] = None,
  229. body: Optional[bytes] = None,
  230. headers: Optional[DictType] = None,
  231. query_params: Optional[DictType] = None,
  232. preload_content: bool = True,
  233. no_body_trace: bool = False,
  234. ) -> BaseHTTPResponse:
  235. """Execute HTTP request."""
  236. creds = self._provider.retrieve() if self._provider else None
  237. url = self._base_url.build(
  238. method=method,
  239. region=region,
  240. bucket_name=bucket_name,
  241. object_name=object_name,
  242. query_params=query_params,
  243. )
  244. headers, date = self._build_headers(url.netloc, headers, body, creds)
  245. if creds:
  246. headers = sign_v4_s3(
  247. method=method,
  248. url=url,
  249. region=region,
  250. headers=headers,
  251. credentials=creds,
  252. content_sha256=cast(str, headers.get("x-amz-content-sha256")),
  253. date=date,
  254. )
  255. if self._trace_stream:
  256. self._trace_stream.write("---------START-HTTP---------\n")
  257. query = ("?" + url.query) if url.query else ""
  258. self._trace_stream.write(f"{method} {url.path}{query} HTTP/1.1\n")
  259. self._trace_stream.write(
  260. headers_to_strings(headers, titled_key=True),
  261. )
  262. self._trace_stream.write("\n")
  263. if not no_body_trace and body is not None:
  264. self._trace_stream.write("\n")
  265. self._trace_stream.write(
  266. body.decode() if isinstance(body, bytes) else str(body),
  267. )
  268. self._trace_stream.write("\n")
  269. self._trace_stream.write("\n")
  270. http_headers = HTTPHeaderDict()
  271. for key, value in (headers or {}).items():
  272. if isinstance(value, (list, tuple)):
  273. for val in value:
  274. http_headers.add(key, val)
  275. else:
  276. http_headers.add(key, value)
  277. response = self._http.urlopen(
  278. method,
  279. urlunsplit(url),
  280. body=body,
  281. headers=http_headers,
  282. preload_content=preload_content,
  283. )
  284. if self._trace_stream:
  285. self._trace_stream.write(f"HTTP/1.1 {response.status}\n")
  286. self._trace_stream.write(
  287. headers_to_strings(response.headers),
  288. )
  289. self._trace_stream.write("\n")
  290. if response.status in [200, 204, 206]:
  291. if self._trace_stream:
  292. if preload_content:
  293. self._trace_stream.write("\n")
  294. self._trace_stream.write(response.data.decode())
  295. self._trace_stream.write("\n")
  296. self._trace_stream.write("----------END-HTTP----------\n")
  297. return response
  298. response.read(cache_content=True)
  299. if not preload_content:
  300. response.release_conn()
  301. if self._trace_stream and method != "HEAD" and response.data:
  302. self._trace_stream.write(response.data.decode())
  303. self._trace_stream.write("\n")
  304. if (
  305. method != "HEAD" and
  306. "application/xml" not in response.headers.get(
  307. "content-type", "",
  308. ).split(";")
  309. ):
  310. if self._trace_stream:
  311. self._trace_stream.write("----------END-HTTP----------\n")
  312. if response.status == 304 and not response.data:
  313. raise ServerError(
  314. f"server failed with HTTP status code {response.status}",
  315. response.status,
  316. )
  317. raise InvalidResponseError(
  318. response.status,
  319. cast(str, response.headers.get("content-type")),
  320. response.data.decode() if response.data else None,
  321. )
  322. if not response.data and method != "HEAD":
  323. if self._trace_stream:
  324. self._trace_stream.write("----------END-HTTP----------\n")
  325. raise InvalidResponseError(
  326. response.status,
  327. response.headers.get("content-type"),
  328. None,
  329. )
  330. response_error = S3Error.fromxml(response) if response.data else None
  331. if self._trace_stream:
  332. self._trace_stream.write("----------END-HTTP----------\n")
  333. error_map = {
  334. 301: lambda: self._handle_redirect_response(
  335. method, response, bucket_name, True,
  336. ),
  337. 307: lambda: self._handle_redirect_response(
  338. method, response, bucket_name, True,
  339. ),
  340. 400: lambda: self._handle_redirect_response(
  341. method, response, bucket_name, True,
  342. ),
  343. 403: lambda: ("AccessDenied", "Access denied"),
  344. 404: lambda: (
  345. ("NoSuchKey", "Object does not exist")
  346. if object_name
  347. else ("NoSuchBucket", "Bucket does not exist")
  348. if bucket_name
  349. else ("ResourceNotFound", "Request resource not found")
  350. ),
  351. 405: lambda: (
  352. "MethodNotAllowed",
  353. "The specified method is not allowed against this resource",
  354. ),
  355. 409: lambda: (
  356. ("NoSuchBucket", "Bucket does not exist")
  357. if bucket_name
  358. else ("ResourceConflict", "Request resource conflicts"),
  359. ),
  360. 501: lambda: (
  361. "MethodNotAllowed",
  362. "The specified method is not allowed against this resource",
  363. ),
  364. }
  365. if not response_error:
  366. func = error_map.get(response.status)
  367. code, message = func() if func else (None, None)
  368. if not code:
  369. raise ServerError(
  370. f"server failed with HTTP status code {response.status}",
  371. response.status,
  372. )
  373. response_error = S3Error(
  374. response=response,
  375. code=cast(str, code),
  376. message=cast(Union[str, None], message),
  377. resource=url.path,
  378. request_id=response.headers.get("x-amz-request-id"),
  379. host_id=response.headers.get("x-amz-id-2"),
  380. bucket_name=bucket_name,
  381. object_name=object_name,
  382. )
  383. if response_error.code in ["NoSuchBucket", "RetryHead"]:
  384. if bucket_name is not None:
  385. self._region_map.pop(bucket_name, None)
  386. raise response_error
  387. def _execute(
  388. self,
  389. method: str,
  390. bucket_name: Optional[str] = None,
  391. object_name: Optional[str] = None,
  392. body: Optional[bytes] = None,
  393. headers: Optional[DictType] = None,
  394. query_params: Optional[DictType] = None,
  395. preload_content: bool = True,
  396. no_body_trace: bool = False,
  397. ) -> BaseHTTPResponse:
  398. """Execute HTTP request."""
  399. region = self._get_region(bucket_name)
  400. try:
  401. return self._url_open(
  402. method,
  403. region,
  404. bucket_name=bucket_name,
  405. object_name=object_name,
  406. body=body,
  407. headers=headers,
  408. query_params=query_params,
  409. preload_content=preload_content,
  410. no_body_trace=no_body_trace,
  411. )
  412. except S3Error as exc:
  413. if exc.code != "RetryHead":
  414. raise
  415. # Retry only once on RetryHead error.
  416. try:
  417. return self._url_open(
  418. method,
  419. region,
  420. bucket_name=bucket_name,
  421. object_name=object_name,
  422. body=body,
  423. headers=headers,
  424. query_params=query_params,
  425. preload_content=preload_content,
  426. no_body_trace=no_body_trace,
  427. )
  428. except S3Error as exc:
  429. if exc.code != "RetryHead":
  430. raise
  431. code, message = self._handle_redirect_response(
  432. method, exc.response, bucket_name,
  433. )
  434. raise exc.copy(cast(str, code), cast(str, message))
  435. def _get_region(self, bucket_name: Optional[str] = None) -> str:
  436. """
  437. Return region of given bucket either from region cache or set in
  438. constructor.
  439. """
  440. if self._base_url.region:
  441. return self._base_url.region
  442. if not bucket_name or not self._provider:
  443. return "us-east-1"
  444. region = self._region_map.get(bucket_name)
  445. if region:
  446. return region
  447. # Execute GetBucketLocation REST API to get region of the bucket.
  448. response = self._url_open(
  449. "GET",
  450. "us-east-1",
  451. bucket_name=bucket_name,
  452. query_params={"location": ""},
  453. )
  454. element = ET.fromstring(response.data.decode())
  455. if not element.text:
  456. region = "us-east-1"
  457. elif element.text == "EU" and self._base_url.is_aws_host:
  458. region = "eu-west-1"
  459. else:
  460. region = element.text
  461. self._region_map[bucket_name] = region
  462. return region
  463. def set_app_info(self, app_name: str, app_version: str):
  464. """
  465. Set your application name and version to user agent header.
  466. :param app_name: Application name.
  467. :param app_version: Application version.
  468. Example::
  469. client.set_app_info('my_app', '1.0.2')
  470. """
  471. if not (app_name and app_version):
  472. raise ValueError("Application name/version cannot be empty.")
  473. self._user_agent = f"{_DEFAULT_USER_AGENT} {app_name}/{app_version}"
  474. def trace_on(self, stream: TextIO):
  475. """
  476. Enable http trace.
  477. :param stream: Stream for writing HTTP call tracing.
  478. """
  479. if not stream:
  480. raise ValueError('Input stream for trace output is invalid.')
  481. # Save new output stream.
  482. self._trace_stream = stream
  483. def trace_off(self):
  484. """
  485. Disable HTTP trace.
  486. """
  487. self._trace_stream = None
  488. def enable_accelerate_endpoint(self):
  489. """Enables accelerate endpoint for Amazon S3 endpoint."""
  490. self._base_url.accelerate_host_flag = True
  491. def disable_accelerate_endpoint(self):
  492. """Disables accelerate endpoint for Amazon S3 endpoint."""
  493. self._base_url.accelerate_host_flag = False
  494. def enable_dualstack_endpoint(self):
  495. """Enables dualstack endpoint for Amazon S3 endpoint."""
  496. self._base_url.dualstack_host_flag = True
  497. def disable_dualstack_endpoint(self):
  498. """Disables dualstack endpoint for Amazon S3 endpoint."""
  499. self._base_url.dualstack_host_flag = False
  500. def enable_virtual_style_endpoint(self):
  501. """Enables virtual style endpoint."""
  502. self._base_url.virtual_style_flag = True
  503. def disable_virtual_style_endpoint(self):
  504. """Disables virtual style endpoint."""
  505. self._base_url.virtual_style_flag = False
  506. def select_object_content(
  507. self,
  508. bucket_name: str,
  509. object_name: str,
  510. request: SelectRequest,
  511. ) -> SelectObjectReader:
  512. """
  513. Select content of an object by SQL expression.
  514. :param bucket_name: Name of the bucket.
  515. :param object_name: Object name in the bucket.
  516. :param request: :class:`SelectRequest <SelectRequest>` object.
  517. :return: A reader contains requested records and progress information.
  518. Example::
  519. with client.select_object_content(
  520. "my-bucket",
  521. "my-object.csv",
  522. SelectRequest(
  523. "select * from S3Object",
  524. CSVInputSerialization(),
  525. CSVOutputSerialization(),
  526. request_progress=True,
  527. ),
  528. ) as result:
  529. for data in result.stream():
  530. print(data.decode())
  531. print(result.stats())
  532. """
  533. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  534. check_object_name(object_name)
  535. if not isinstance(request, SelectRequest):
  536. raise ValueError("request must be SelectRequest type")
  537. body = marshal(request)
  538. response = self._execute(
  539. "POST",
  540. bucket_name=bucket_name,
  541. object_name=object_name,
  542. body=body,
  543. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  544. query_params={"select": "", "select-type": "2"},
  545. preload_content=False,
  546. )
  547. return SelectObjectReader(response)
  548. def make_bucket(
  549. self,
  550. bucket_name: str,
  551. location: Optional[str] = None,
  552. object_lock: bool = False,
  553. ):
  554. """
  555. Create a bucket with region and object lock.
  556. :param bucket_name: Name of the bucket.
  557. :param location: Region in which the bucket will be created.
  558. :param object_lock: Flag to set object-lock feature.
  559. Examples::
  560. # Create bucket.
  561. client.make_bucket("my-bucket")
  562. # Create bucket on specific region.
  563. client.make_bucket("my-bucket", "us-west-1")
  564. # Create bucket with object-lock feature on specific region.
  565. client.make_bucket("my-bucket", "eu-west-2", object_lock=True)
  566. """
  567. check_bucket_name(bucket_name, True,
  568. s3_check=self._base_url.is_aws_host)
  569. if self._base_url.region:
  570. # Error out if region does not match with region passed via
  571. # constructor.
  572. if location and self._base_url.region != location:
  573. raise ValueError(
  574. f"region must be {self._base_url.region}, "
  575. f"but passed {location}"
  576. )
  577. location = self._base_url.region or location or "us-east-1"
  578. headers: Optional[DictType] = (
  579. {"x-amz-bucket-object-lock-enabled": "true"}
  580. if object_lock else None
  581. )
  582. body = None
  583. if location != "us-east-1":
  584. element = Element("CreateBucketConfiguration")
  585. SubElement(element, "LocationConstraint", location)
  586. body = getbytes(element)
  587. self._url_open(
  588. "PUT",
  589. location,
  590. bucket_name=bucket_name,
  591. body=body,
  592. headers=headers,
  593. )
  594. self._region_map[bucket_name] = location
  595. def list_buckets(self) -> list[Bucket]:
  596. """
  597. List information of all accessible buckets.
  598. :return: List of :class:`Bucket <Bucket>` object.
  599. Example::
  600. buckets = client.list_buckets()
  601. for bucket in buckets:
  602. print(bucket.name, bucket.creation_date)
  603. """
  604. response = self._execute("GET")
  605. result = unmarshal(ListAllMyBucketsResult, response.data.decode())
  606. return result.buckets
  607. def bucket_exists(self, bucket_name: str) -> bool:
  608. """
  609. Check if a bucket exists.
  610. :param bucket_name: Name of the bucket.
  611. :return: True if the bucket exists.
  612. Example::
  613. if client.bucket_exists("my-bucket"):
  614. print("my-bucket exists")
  615. else:
  616. print("my-bucket does not exist")
  617. """
  618. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  619. try:
  620. self._execute("HEAD", bucket_name)
  621. return True
  622. except S3Error as exc:
  623. if exc.code != "NoSuchBucket":
  624. raise
  625. return False
  626. def remove_bucket(self, bucket_name: str):
  627. """
  628. Remove an empty bucket.
  629. :param bucket_name: Name of the bucket.
  630. Example::
  631. client.remove_bucket("my-bucket")
  632. """
  633. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  634. self._execute("DELETE", bucket_name)
  635. self._region_map.pop(bucket_name, None)
  636. def get_bucket_policy(self, bucket_name: str) -> str:
  637. """
  638. Get bucket policy configuration of a bucket.
  639. :param bucket_name: Name of the bucket.
  640. :return: Bucket policy configuration as JSON string.
  641. Example::
  642. policy = client.get_bucket_policy("my-bucket")
  643. """
  644. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  645. response = self._execute(
  646. "GET", bucket_name, query_params={"policy": ""},
  647. )
  648. return response.data.decode()
  649. def delete_bucket_policy(self, bucket_name: str):
  650. """
  651. Delete bucket policy configuration of a bucket.
  652. :param bucket_name: Name of the bucket.
  653. Example::
  654. client.delete_bucket_policy("my-bucket")
  655. """
  656. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  657. self._execute("DELETE", bucket_name, query_params={"policy": ""})
  658. def set_bucket_policy(self, bucket_name: str, policy: str | bytes):
  659. """
  660. Set bucket policy configuration to a bucket.
  661. :param bucket_name: Name of the bucket.
  662. :param policy: Bucket policy configuration as JSON string.
  663. Example::
  664. client.set_bucket_policy("my-bucket", policy)
  665. """
  666. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  667. is_valid_policy_type(policy)
  668. self._execute(
  669. "PUT",
  670. bucket_name,
  671. body=policy if isinstance(policy, bytes) else policy.encode(),
  672. headers={"Content-MD5": cast(str, md5sum_hash(policy))},
  673. query_params={"policy": ""},
  674. )
  675. def get_bucket_notification(self, bucket_name: str) -> NotificationConfig:
  676. """
  677. Get notification configuration of a bucket.
  678. :param bucket_name: Name of the bucket.
  679. :return: :class:`NotificationConfig <NotificationConfig>` object.
  680. Example::
  681. config = client.get_bucket_notification("my-bucket")
  682. """
  683. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  684. response = self._execute(
  685. "GET", bucket_name, query_params={"notification": ""},
  686. )
  687. return unmarshal(NotificationConfig, response.data.decode())
  688. def set_bucket_notification(
  689. self,
  690. bucket_name: str,
  691. config: NotificationConfig,
  692. ):
  693. """
  694. Set notification configuration of a bucket.
  695. :param bucket_name: Name of the bucket.
  696. :param config: class:`NotificationConfig <NotificationConfig>` object.
  697. Example::
  698. config = NotificationConfig(
  699. queue_config_list=[
  700. QueueConfig(
  701. "QUEUE-ARN-OF-THIS-BUCKET",
  702. ["s3:ObjectCreated:*"],
  703. config_id="1",
  704. prefix_filter_rule=PrefixFilterRule("abc"),
  705. ),
  706. ],
  707. )
  708. client.set_bucket_notification("my-bucket", config)
  709. """
  710. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  711. if not isinstance(config, NotificationConfig):
  712. raise ValueError("config must be NotificationConfig type")
  713. body = marshal(config)
  714. self._execute(
  715. "PUT",
  716. bucket_name,
  717. body=body,
  718. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  719. query_params={"notification": ""},
  720. )
  721. def delete_bucket_notification(self, bucket_name: str):
  722. """
  723. Delete notification configuration of a bucket. On success, S3 service
  724. stops notification of events previously set of the bucket.
  725. :param bucket_name: Name of the bucket.
  726. Example::
  727. client.delete_bucket_notification("my-bucket")
  728. """
  729. self.set_bucket_notification(bucket_name, NotificationConfig())
  730. def set_bucket_encryption(self, bucket_name: str, config: SSEConfig):
  731. """
  732. Set encryption configuration of a bucket.
  733. :param bucket_name: Name of the bucket.
  734. :param config: :class:`SSEConfig <SSEConfig>` object.
  735. Example::
  736. client.set_bucket_encryption(
  737. "my-bucket", SSEConfig(Rule.new_sse_s3_rule()),
  738. )
  739. """
  740. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  741. if not isinstance(config, SSEConfig):
  742. raise ValueError("config must be SSEConfig type")
  743. body = marshal(config)
  744. self._execute(
  745. "PUT",
  746. bucket_name,
  747. body=body,
  748. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  749. query_params={"encryption": ""},
  750. )
  751. def get_bucket_encryption(self, bucket_name: str) -> Optional[SSEConfig]:
  752. """
  753. Get encryption configuration of a bucket.
  754. :param bucket_name: Name of the bucket.
  755. :return: :class:`SSEConfig <SSEConfig>` object.
  756. Example::
  757. config = client.get_bucket_encryption("my-bucket")
  758. """
  759. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  760. try:
  761. response = self._execute(
  762. "GET",
  763. bucket_name,
  764. query_params={"encryption": ""},
  765. )
  766. return unmarshal(SSEConfig, response.data.decode())
  767. except S3Error as exc:
  768. if exc.code != "ServerSideEncryptionConfigurationNotFoundError":
  769. raise
  770. return None
  771. def delete_bucket_encryption(self, bucket_name: str):
  772. """
  773. Delete encryption configuration of a bucket.
  774. :param bucket_name: Name of the bucket.
  775. Example::
  776. client.delete_bucket_encryption("my-bucket")
  777. """
  778. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  779. try:
  780. self._execute(
  781. "DELETE",
  782. bucket_name,
  783. query_params={"encryption": ""},
  784. )
  785. except S3Error as exc:
  786. if exc.code != "ServerSideEncryptionConfigurationNotFoundError":
  787. raise
  788. def listen_bucket_notification(
  789. self,
  790. bucket_name: str,
  791. prefix: str = "",
  792. suffix: str = "",
  793. events: tuple[str, ...] = ('s3:ObjectCreated:*',
  794. 's3:ObjectRemoved:*',
  795. 's3:ObjectAccessed:*'),
  796. ) -> EventIterable:
  797. """
  798. Listen events of object prefix and suffix of a bucket. Caller should
  799. iterate returned iterator to read new events.
  800. :param bucket_name: Name of the bucket.
  801. :param prefix: Listen events of object starts with prefix.
  802. :param suffix: Listen events of object ends with suffix.
  803. :param events: Events to listen.
  804. :return: Iterator of event records as :dict:.
  805. Example::
  806. with client.listen_bucket_notification(
  807. "my-bucket",
  808. prefix="my-prefix/",
  809. events=["s3:ObjectCreated:*", "s3:ObjectRemoved:*"],
  810. ) as events:
  811. for event in events:
  812. print(event)
  813. """
  814. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  815. if self._base_url.is_aws_host:
  816. raise ValueError(
  817. "ListenBucketNotification API is not supported in Amazon S3",
  818. )
  819. return EventIterable(
  820. lambda: self._execute(
  821. "GET",
  822. bucket_name,
  823. query_params={
  824. "prefix": prefix or "",
  825. "suffix": suffix or "",
  826. "events": cast(Tuple[str], events),
  827. },
  828. preload_content=False,
  829. ),
  830. )
  831. def set_bucket_versioning(
  832. self,
  833. bucket_name: str,
  834. config: VersioningConfig,
  835. ):
  836. """
  837. Set versioning configuration to a bucket.
  838. :param bucket_name: Name of the bucket.
  839. :param config: :class:`VersioningConfig <VersioningConfig>`.
  840. Example::
  841. client.set_bucket_versioning(
  842. "my-bucket", VersioningConfig(ENABLED),
  843. )
  844. """
  845. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  846. if not isinstance(config, VersioningConfig):
  847. raise ValueError("config must be VersioningConfig type")
  848. body = marshal(config)
  849. self._execute(
  850. "PUT",
  851. bucket_name,
  852. body=body,
  853. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  854. query_params={"versioning": ""},
  855. )
  856. def get_bucket_versioning(self, bucket_name: str) -> VersioningConfig:
  857. """
  858. Get versioning configuration of a bucket.
  859. :param bucket_name: Name of the bucket.
  860. :return: :class:`VersioningConfig <VersioningConfig>`.
  861. Example::
  862. config = client.get_bucket_versioning("my-bucket")
  863. print(config.status)
  864. """
  865. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  866. response = self._execute(
  867. "GET",
  868. bucket_name,
  869. query_params={"versioning": ""},
  870. )
  871. return unmarshal(VersioningConfig, response.data.decode())
  872. def fput_object(
  873. self,
  874. bucket_name: str,
  875. object_name: str,
  876. file_path: str,
  877. content_type: str = "application/octet-stream",
  878. metadata: Optional[DictType] = None,
  879. sse: Optional[Sse] = None,
  880. progress: Optional[ProgressType] = None,
  881. part_size: int = 0,
  882. num_parallel_uploads: int = 3,
  883. tags: Optional[Tags] = None,
  884. retention: Optional[Retention] = None,
  885. legal_hold: bool = False,
  886. ) -> ObjectWriteResult:
  887. """
  888. Uploads data from a file to an object in a bucket.
  889. :param bucket_name: Name of the bucket.
  890. :param object_name: Object name in the bucket.
  891. :param file_path: Name of file to upload.
  892. :param content_type: Content type of the object.
  893. :param metadata: Any additional metadata to be uploaded along
  894. with your PUT request.
  895. :param sse: Server-side encryption.
  896. :param progress: A progress object
  897. :param part_size: Multipart part size
  898. :param num_parallel_uploads: Number of parallel uploads.
  899. :param tags: :class:`Tags` for the object.
  900. :param retention: :class:`Retention` configuration object.
  901. :param legal_hold: Flag to set legal hold for the object.
  902. :return: :class:`ObjectWriteResult` object.
  903. Example::
  904. # Upload data.
  905. result = client.fput_object(
  906. "my-bucket", "my-object", "my-filename",
  907. )
  908. # Upload data with metadata.
  909. result = client.fput_object(
  910. "my-bucket", "my-object", "my-filename",
  911. metadata={"My-Project": "one"},
  912. )
  913. # Upload data with tags, retention and legal-hold.
  914. date = datetime.utcnow().replace(
  915. hour=0, minute=0, second=0, microsecond=0,
  916. ) + timedelta(days=30)
  917. tags = Tags(for_object=True)
  918. tags["User"] = "jsmith"
  919. result = client.fput_object(
  920. "my-bucket", "my-object", "my-filename",
  921. tags=tags,
  922. retention=Retention(GOVERNANCE, date),
  923. legal_hold=True,
  924. )
  925. """
  926. file_size = os.stat(file_path).st_size
  927. with open(file_path, "rb") as file_data:
  928. return self.put_object(
  929. bucket_name,
  930. object_name,
  931. file_data,
  932. file_size,
  933. content_type=content_type,
  934. metadata=cast(Union[DictType, None], metadata),
  935. sse=sse,
  936. progress=progress,
  937. part_size=part_size,
  938. num_parallel_uploads=num_parallel_uploads,
  939. tags=tags,
  940. retention=retention,
  941. legal_hold=legal_hold,
  942. )
  943. def fget_object(
  944. self,
  945. bucket_name: str,
  946. object_name: str,
  947. file_path: str,
  948. request_headers: Optional[DictType] = None,
  949. ssec: Optional[SseCustomerKey] = None,
  950. version_id: Optional[str] = None,
  951. extra_query_params: Optional[DictType] = None,
  952. tmp_file_path: Optional[str] = None,
  953. progress: Optional[ProgressType] = None,
  954. ):
  955. """
  956. Downloads data of an object to file.
  957. :param bucket_name: Name of the bucket.
  958. :param object_name: Object name in the bucket.
  959. :param file_path: Name of file to download.
  960. :param request_headers: Any additional headers to be added with GET
  961. request.
  962. :param ssec: Server-side encryption customer key.
  963. :param version_id: Version-ID of the object.
  964. :param extra_query_params: Extra query parameters for advanced usage.
  965. :param tmp_file_path: Path to a temporary file.
  966. :param progress: A progress object
  967. :return: Object information.
  968. Example::
  969. # Download data of an object.
  970. client.fget_object("my-bucket", "my-object", "my-filename")
  971. # Download data of an object of version-ID.
  972. client.fget_object(
  973. "my-bucket", "my-object", "my-filename",
  974. version_id="dfbd25b3-abec-4184-a4e8-5a35a5c1174d",
  975. )
  976. # Download data of an SSE-C encrypted object.
  977. client.fget_object(
  978. "my-bucket", "my-object", "my-filename",
  979. ssec=SseCustomerKey(b"32byteslongsecretkeymustprovided"),
  980. )
  981. """
  982. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  983. check_object_name(object_name)
  984. if os.path.isdir(file_path):
  985. raise ValueError(f"file {file_path} is a directory")
  986. # Create top level directory if needed.
  987. makedirs(os.path.dirname(file_path))
  988. stat = self.stat_object(
  989. bucket_name,
  990. object_name,
  991. ssec,
  992. version_id=version_id,
  993. extra_headers=request_headers,
  994. )
  995. etag = queryencode(cast(str, stat.etag))
  996. # Write to a temporary file "file_path.part.minio" before saving.
  997. tmp_file_path = (
  998. tmp_file_path or f"{file_path}.{etag}.part.minio"
  999. )
  1000. response = None
  1001. try:
  1002. response = self.get_object(
  1003. bucket_name,
  1004. object_name,
  1005. request_headers=request_headers,
  1006. ssec=ssec,
  1007. version_id=version_id,
  1008. extra_query_params=extra_query_params,
  1009. )
  1010. if progress:
  1011. # Set progress bar length and object name before upload
  1012. length = int(response.headers.get('content-length', 0))
  1013. progress.set_meta(object_name=object_name, total_length=length)
  1014. with open(tmp_file_path, "wb") as tmp_file:
  1015. for data in response.stream(amt=1024 * 1024):
  1016. size = tmp_file.write(data)
  1017. if progress:
  1018. progress.update(size)
  1019. if os.path.exists(file_path):
  1020. os.remove(file_path) # For windows compatibility.
  1021. os.rename(tmp_file_path, file_path)
  1022. return stat
  1023. finally:
  1024. if response:
  1025. response.close()
  1026. response.release_conn()
  1027. def get_object(
  1028. self,
  1029. bucket_name: str,
  1030. object_name: str,
  1031. offset: int = 0,
  1032. length: int = 0,
  1033. request_headers: Optional[DictType] = None,
  1034. ssec: Optional[SseCustomerKey] = None,
  1035. version_id: Optional[str] = None,
  1036. extra_query_params: Optional[DictType] = None,
  1037. ) -> BaseHTTPResponse:
  1038. """
  1039. Get data of an object. Returned response should be closed after use to
  1040. release network resources. To reuse the connection, it's required to
  1041. call `response.release_conn()` explicitly.
  1042. :param bucket_name: Name of the bucket.
  1043. :param object_name: Object name in the bucket.
  1044. :param offset: Start byte position of object data.
  1045. :param length: Number of bytes of object data from offset.
  1046. :param request_headers: Any additional headers to be added with GET
  1047. request.
  1048. :param ssec: Server-side encryption customer key.
  1049. :param version_id: Version-ID of the object.
  1050. :param extra_query_params: Extra query parameters for advanced usage.
  1051. :return: :class:`urllib3.response.BaseHTTPResponse` object.
  1052. Example::
  1053. # Get data of an object.
  1054. response = None
  1055. try:
  1056. response = client.get_object("my-bucket", "my-object")
  1057. # Read data from response.
  1058. finally:
  1059. if response:
  1060. response.close()
  1061. response.release_conn()
  1062. # Get data of an object of version-ID.
  1063. response = None
  1064. try:
  1065. response = client.get_object(
  1066. "my-bucket", "my-object",
  1067. version_id="dfbd25b3-abec-4184-a4e8-5a35a5c1174d",
  1068. )
  1069. # Read data from response.
  1070. finally:
  1071. if response:
  1072. response.close()
  1073. response.release_conn()
  1074. # Get data of an object from offset and length.
  1075. response = None
  1076. try:
  1077. response = client.get_object(
  1078. "my-bucket", "my-object", offset=512, length=1024,
  1079. )
  1080. # Read data from response.
  1081. finally:
  1082. if response:
  1083. response.close()
  1084. response.release_conn()
  1085. # Get data of an SSE-C encrypted object.
  1086. response = None
  1087. try:
  1088. response = client.get_object(
  1089. "my-bucket", "my-object",
  1090. ssec=SseCustomerKey(b"32byteslongsecretkeymustprovided"),
  1091. )
  1092. # Read data from response.
  1093. finally:
  1094. if response:
  1095. response.close()
  1096. response.release_conn()
  1097. """
  1098. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  1099. check_object_name(object_name)
  1100. check_ssec(ssec)
  1101. headers = cast(DictType, ssec.headers() if ssec else {})
  1102. headers.update(request_headers or {})
  1103. if offset or length:
  1104. end = (offset + length - 1) if length else ""
  1105. headers['Range'] = f"bytes={offset}-{end}"
  1106. if version_id:
  1107. extra_query_params = extra_query_params or {}
  1108. extra_query_params["versionId"] = version_id
  1109. return self._execute(
  1110. "GET",
  1111. bucket_name,
  1112. object_name,
  1113. headers=cast(DictType, headers),
  1114. query_params=extra_query_params,
  1115. preload_content=False,
  1116. )
  1117. def prompt_object(
  1118. self,
  1119. bucket_name: str,
  1120. object_name: str,
  1121. prompt: str,
  1122. lambda_arn: Optional[str] = None,
  1123. request_headers: Optional[DictType] = None,
  1124. ssec: Optional[SseCustomerKey] = None,
  1125. version_id: Optional[str] = None,
  1126. **kwargs: Optional[Any],
  1127. ) -> BaseHTTPResponse:
  1128. """
  1129. Prompt an object using natural language.
  1130. :param bucket_name: Name of the bucket.
  1131. :param object_name: Object name in the bucket.
  1132. :param prompt: Prompt the Object to interact with the AI model.
  1133. request.
  1134. :param lambda_arn: Lambda ARN to use for prompt.
  1135. :param request_headers: Any additional headers to be added with POST
  1136. :param ssec: Server-side encryption customer key.
  1137. :param version_id: Version-ID of the object.
  1138. :param kwargs: Extra parameters for advanced usage.
  1139. :return: :class:`urllib3.response.BaseHTTPResponse` object.
  1140. Example::
  1141. # prompt an object.
  1142. response = None
  1143. try:
  1144. response = client.get_object(
  1145. "my-bucket", "my-object",
  1146. "Describe the object for me")
  1147. # Read data from response.
  1148. finally:
  1149. if response:
  1150. response.close()
  1151. response.release_conn()
  1152. """
  1153. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  1154. check_object_name(object_name)
  1155. check_ssec(ssec)
  1156. headers = cast(DictType, ssec.headers() if ssec else {})
  1157. headers.update(request_headers or {})
  1158. extra_query_params = {"lambdaArn": lambda_arn or ""}
  1159. if version_id:
  1160. extra_query_params["versionId"] = version_id
  1161. prompt_body = kwargs
  1162. prompt_body["prompt"] = prompt
  1163. body = json.dumps(prompt_body)
  1164. return self._execute(
  1165. "POST",
  1166. bucket_name,
  1167. object_name,
  1168. headers=cast(DictType, headers),
  1169. query_params=cast(DictType, extra_query_params),
  1170. body=body.encode(),
  1171. preload_content=False,
  1172. )
  1173. def copy_object(
  1174. self,
  1175. bucket_name: str,
  1176. object_name: str,
  1177. source: CopySource,
  1178. sse: Optional[Sse] = None,
  1179. metadata: Optional[DictType] = None,
  1180. tags: Optional[Tags] = None,
  1181. retention: Optional[Retention] = None,
  1182. legal_hold: bool = False,
  1183. metadata_directive: Optional[str] = None,
  1184. tagging_directive: Optional[str] = None,
  1185. ) -> ObjectWriteResult:
  1186. """
  1187. Create an object by server-side copying data from another object.
  1188. In this API maximum supported source object size is 5GiB.
  1189. :param bucket_name: Name of the bucket.
  1190. :param object_name: Object name in the bucket.
  1191. :param source: :class:`CopySource` object.
  1192. :param sse: Server-side encryption of destination object.
  1193. :param metadata: Any user-defined metadata to be copied along with
  1194. destination object.
  1195. :param tags: Tags for destination object.
  1196. :param retention: :class:`Retention` configuration object.
  1197. :param legal_hold: Flag to set legal hold for destination object.
  1198. :param metadata_directive: Directive used to handle user metadata for
  1199. destination object.
  1200. :param tagging_directive: Directive used to handle tags for destination
  1201. object.
  1202. :return: :class:`ObjectWriteResult <ObjectWriteResult>` object.
  1203. Example::
  1204. # copy an object from a bucket to another.
  1205. result = client.copy_object(
  1206. "my-bucket",
  1207. "my-object",
  1208. CopySource("my-sourcebucket", "my-sourceobject"),
  1209. )
  1210. print(result.object_name, result.version_id)
  1211. # copy an object with condition.
  1212. result = client.copy_object(
  1213. "my-bucket",
  1214. "my-object",
  1215. CopySource(
  1216. "my-sourcebucket",
  1217. "my-sourceobject",
  1218. modified_since=datetime(2014, 4, 1, tzinfo=timezone.utc),
  1219. ),
  1220. )
  1221. print(result.object_name, result.version_id)
  1222. # copy an object from a bucket with replacing metadata.
  1223. metadata = {"test_meta_key": "test_meta_value"}
  1224. result = client.copy_object(
  1225. "my-bucket",
  1226. "my-object",
  1227. CopySource("my-sourcebucket", "my-sourceobject"),
  1228. metadata=metadata,
  1229. metadata_directive=REPLACE,
  1230. )
  1231. print(result.object_name, result.version_id)
  1232. """
  1233. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  1234. check_object_name(object_name)
  1235. if not isinstance(source, CopySource):
  1236. raise ValueError("source must be CopySource type")
  1237. check_sse(sse)
  1238. if tags is not None and not isinstance(tags, Tags):
  1239. raise ValueError("tags must be Tags type")
  1240. if retention is not None and not isinstance(retention, Retention):
  1241. raise ValueError("retention must be Retention type")
  1242. if (
  1243. metadata_directive is not None and
  1244. metadata_directive not in [COPY, REPLACE]
  1245. ):
  1246. raise ValueError(f"metadata directive must be {COPY} or {REPLACE}")
  1247. if (
  1248. tagging_directive is not None and
  1249. tagging_directive not in [COPY, REPLACE]
  1250. ):
  1251. raise ValueError(f"tagging directive must be {COPY} or {REPLACE}")
  1252. size = -1
  1253. if source.offset is None and source.length is None:
  1254. stat = self.stat_object(
  1255. source.bucket_name,
  1256. source.object_name,
  1257. version_id=source.version_id,
  1258. ssec=source.ssec,
  1259. )
  1260. size = cast(int, stat.size)
  1261. if (
  1262. source.offset is not None or
  1263. source.length is not None or
  1264. size > MAX_PART_SIZE
  1265. ):
  1266. if metadata_directive == COPY:
  1267. raise ValueError(
  1268. "COPY metadata directive is not applicable to source "
  1269. "object size greater than 5 GiB",
  1270. )
  1271. if tagging_directive == COPY:
  1272. raise ValueError(
  1273. "COPY tagging directive is not applicable to source "
  1274. "object size greater than 5 GiB"
  1275. )
  1276. return self.compose_object(
  1277. bucket_name, object_name, [ComposeSource.of(source)],
  1278. sse=sse, metadata=metadata, tags=tags, retention=retention,
  1279. legal_hold=legal_hold,
  1280. )
  1281. headers = genheaders(
  1282. metadata,
  1283. sse,
  1284. tags,
  1285. retention,
  1286. legal_hold,
  1287. )
  1288. if metadata_directive:
  1289. headers["x-amz-metadata-directive"] = metadata_directive
  1290. if tagging_directive:
  1291. headers["x-amz-tagging-directive"] = tagging_directive
  1292. headers.update(source.gen_copy_headers())
  1293. response = self._execute(
  1294. "PUT",
  1295. bucket_name,
  1296. object_name=object_name,
  1297. headers=headers,
  1298. )
  1299. etag, last_modified = parse_copy_object(response)
  1300. return ObjectWriteResult(
  1301. bucket_name,
  1302. object_name,
  1303. response.headers.get("x-amz-version-id"),
  1304. etag,
  1305. response.headers,
  1306. last_modified=last_modified,
  1307. )
  1308. def _calc_part_count(self, sources: list[ComposeSource]) -> int:
  1309. """Calculate part count."""
  1310. object_size = 0
  1311. part_count = 0
  1312. i = 0
  1313. for src in sources:
  1314. i += 1
  1315. stat = self.stat_object(
  1316. src.bucket_name,
  1317. src.object_name,
  1318. version_id=src.version_id,
  1319. ssec=src.ssec,
  1320. )
  1321. src.build_headers(cast(int, stat.size), cast(str, stat.etag))
  1322. size = cast(int, stat.size)
  1323. if src.length is not None:
  1324. size = src.length
  1325. elif src.offset is not None:
  1326. size -= src.offset
  1327. if (
  1328. size < MIN_PART_SIZE and
  1329. len(sources) != 1 and
  1330. i != len(sources)
  1331. ):
  1332. raise ValueError(
  1333. f"source {src.bucket_name}/{src.object_name}: size {size} "
  1334. f"must be greater than {MIN_PART_SIZE}"
  1335. )
  1336. object_size += size
  1337. if object_size > MAX_MULTIPART_OBJECT_SIZE:
  1338. raise ValueError(
  1339. f"destination object size must be less than "
  1340. f"{MAX_MULTIPART_OBJECT_SIZE}"
  1341. )
  1342. if size > MAX_PART_SIZE:
  1343. count = int(size / MAX_PART_SIZE)
  1344. last_part_size = size - (count * MAX_PART_SIZE)
  1345. if last_part_size > 0:
  1346. count += 1
  1347. else:
  1348. last_part_size = MAX_PART_SIZE
  1349. if (
  1350. last_part_size < MIN_PART_SIZE and
  1351. len(sources) != 1 and
  1352. i != len(sources)
  1353. ):
  1354. raise ValueError(
  1355. f"source {src.bucket_name}/{src.object_name}: "
  1356. f"for multipart split upload of {size}, "
  1357. f"last part size is less than {MIN_PART_SIZE}"
  1358. )
  1359. part_count += count
  1360. else:
  1361. part_count += 1
  1362. if part_count > MAX_MULTIPART_COUNT:
  1363. raise ValueError(
  1364. f"Compose sources create more than allowed multipart "
  1365. f"count {MAX_MULTIPART_COUNT}"
  1366. )
  1367. return part_count
  1368. def _upload_part_copy(
  1369. self,
  1370. bucket_name: str,
  1371. object_name: str,
  1372. upload_id: str,
  1373. part_number: int,
  1374. headers: DictType,
  1375. ) -> tuple[str, Optional[datetime]]:
  1376. """Execute UploadPartCopy S3 API."""
  1377. response = self._execute(
  1378. "PUT",
  1379. bucket_name,
  1380. object_name,
  1381. headers=headers,
  1382. query_params={
  1383. "partNumber": str(part_number),
  1384. "uploadId": upload_id,
  1385. },
  1386. )
  1387. return parse_copy_object(response)
  1388. def compose_object(
  1389. self,
  1390. bucket_name: str,
  1391. object_name: str,
  1392. sources: list[ComposeSource],
  1393. sse: Optional[Sse] = None,
  1394. metadata: Optional[DictType] = None,
  1395. tags: Optional[Tags] = None,
  1396. retention: Optional[Retention] = None,
  1397. legal_hold: bool = False,
  1398. ) -> ObjectWriteResult:
  1399. """
  1400. Create an object by combining data from different source objects using
  1401. server-side copy.
  1402. :param bucket_name: Name of the bucket.
  1403. :param object_name: Object name in the bucket.
  1404. :param sources: List of :class:`ComposeSource` object.
  1405. :param sse: Server-side encryption of destination object.
  1406. :param metadata: Any user-defined metadata to be copied along with
  1407. destination object.
  1408. :param tags: Tags for destination object.
  1409. :param retention: :class:`Retention` configuration object.
  1410. :param legal_hold: Flag to set legal hold for destination object.
  1411. :return: :class:`ObjectWriteResult <ObjectWriteResult>` object.
  1412. Example::
  1413. sources = [
  1414. ComposeSource("my-job-bucket", "my-object-part-one"),
  1415. ComposeSource("my-job-bucket", "my-object-part-two"),
  1416. ComposeSource("my-job-bucket", "my-object-part-three"),
  1417. ]
  1418. # Create my-bucket/my-object by combining source object
  1419. # list.
  1420. result = client.compose_object("my-bucket", "my-object", sources)
  1421. print(result.object_name, result.version_id)
  1422. # Create my-bucket/my-object with user metadata by combining
  1423. # source object list.
  1424. result = client.compose_object(
  1425. "my-bucket",
  1426. "my-object",
  1427. sources,
  1428. metadata={"test_meta_key": "test_meta_value"},
  1429. )
  1430. print(result.object_name, result.version_id)
  1431. # Create my-bucket/my-object with user metadata and
  1432. # server-side encryption by combining source object list.
  1433. client.compose_object(
  1434. "my-bucket", "my-object", sources, sse=SseS3(),
  1435. )
  1436. print(result.object_name, result.version_id)
  1437. """
  1438. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  1439. check_object_name(object_name)
  1440. if not isinstance(sources, (list, tuple)) or not sources:
  1441. raise ValueError("sources must be non-empty list or tuple type")
  1442. i = 0
  1443. for src in sources:
  1444. if not isinstance(src, ComposeSource):
  1445. raise ValueError(f"sources[{i}] must be ComposeSource type")
  1446. i += 1
  1447. check_sse(sse)
  1448. if tags is not None and not isinstance(tags, Tags):
  1449. raise ValueError("tags must be Tags type")
  1450. if retention is not None and not isinstance(retention, Retention):
  1451. raise ValueError("retention must be Retention type")
  1452. part_count = self._calc_part_count(sources)
  1453. if (
  1454. part_count == 1 and
  1455. sources[0].offset is None and
  1456. sources[0].length is None
  1457. ):
  1458. return self.copy_object(
  1459. bucket_name, object_name, CopySource.of(sources[0]),
  1460. sse=sse, metadata=metadata, tags=tags, retention=retention,
  1461. legal_hold=legal_hold,
  1462. metadata_directive=REPLACE if metadata else None,
  1463. tagging_directive=REPLACE if tags else None,
  1464. )
  1465. headers = genheaders(metadata, sse, tags, retention, legal_hold)
  1466. upload_id = self._create_multipart_upload(
  1467. bucket_name, object_name, headers,
  1468. )
  1469. ssec_headers = sse.headers() if isinstance(sse, SseCustomerKey) else {}
  1470. try:
  1471. part_number = 0
  1472. total_parts = []
  1473. for src in sources:
  1474. size = cast(int, src.object_size)
  1475. if src.length is not None:
  1476. size = src.length
  1477. elif src.offset is not None:
  1478. size -= src.offset
  1479. offset = src.offset or 0
  1480. headers = cast(DictType, src.headers)
  1481. headers.update(ssec_headers)
  1482. if size <= MAX_PART_SIZE:
  1483. part_number += 1
  1484. if src.length is not None:
  1485. headers["x-amz-copy-source-range"] = (
  1486. f"bytes={offset}-{offset + src.length - 1}"
  1487. )
  1488. elif src.offset is not None:
  1489. headers["x-amz-copy-source-range"] = (
  1490. f"bytes={offset}-{offset + size - 1}"
  1491. )
  1492. etag, _ = self._upload_part_copy(
  1493. bucket_name,
  1494. object_name,
  1495. upload_id,
  1496. part_number,
  1497. headers,
  1498. )
  1499. total_parts.append(Part(part_number, etag))
  1500. continue
  1501. while size > 0:
  1502. part_number += 1
  1503. length = size if size < MAX_PART_SIZE else MAX_PART_SIZE
  1504. end_bytes = offset + length - 1
  1505. headers_copy = headers.copy()
  1506. headers_copy["x-amz-copy-source-range"] = (
  1507. f"bytes={offset}-{end_bytes}"
  1508. )
  1509. etag, _ = self._upload_part_copy(
  1510. bucket_name,
  1511. object_name,
  1512. upload_id,
  1513. part_number,
  1514. headers_copy,
  1515. )
  1516. total_parts.append(Part(part_number, etag))
  1517. offset += length
  1518. size -= length
  1519. result = self._complete_multipart_upload(
  1520. bucket_name, object_name, upload_id, total_parts,
  1521. sse if isinstance(sse, SseCustomerKey) else None,
  1522. )
  1523. return ObjectWriteResult(
  1524. cast(str, result.bucket_name),
  1525. cast(str, result.object_name),
  1526. result.version_id,
  1527. result.etag,
  1528. result.http_headers,
  1529. location=result.location,
  1530. )
  1531. except Exception as exc:
  1532. if upload_id:
  1533. self._abort_multipart_upload(
  1534. bucket_name, object_name, upload_id,
  1535. )
  1536. raise exc
  1537. def _abort_multipart_upload(
  1538. self,
  1539. bucket_name: str,
  1540. object_name: str,
  1541. upload_id: str,
  1542. ):
  1543. """Execute AbortMultipartUpload S3 API."""
  1544. self._execute(
  1545. "DELETE",
  1546. bucket_name,
  1547. object_name,
  1548. query_params={'uploadId': upload_id},
  1549. )
  1550. def _complete_multipart_upload(
  1551. self,
  1552. bucket_name: str,
  1553. object_name: str,
  1554. upload_id: str,
  1555. parts: list[Part],
  1556. ssec: Optional[SseCustomerKey] = None,
  1557. ) -> CompleteMultipartUploadResult:
  1558. """Execute CompleteMultipartUpload S3 API."""
  1559. element = Element("CompleteMultipartUpload")
  1560. for part in parts:
  1561. tag = SubElement(element, "Part")
  1562. SubElement(tag, "PartNumber", str(part.part_number))
  1563. SubElement(tag, "ETag", '"' + part.etag + '"')
  1564. body = getbytes(element)
  1565. headers: DictType = {
  1566. "Content-Type": 'application/xml',
  1567. "Content-MD5": cast(str, md5sum_hash(body)),
  1568. }
  1569. if ssec:
  1570. headers.update(ssec.headers())
  1571. response = self._execute(
  1572. "POST",
  1573. bucket_name,
  1574. object_name,
  1575. body=body,
  1576. headers=headers,
  1577. query_params={'uploadId': upload_id},
  1578. )
  1579. return CompleteMultipartUploadResult(response)
  1580. def _create_multipart_upload(
  1581. self,
  1582. bucket_name: str,
  1583. object_name: str,
  1584. headers: DictType,
  1585. ) -> str:
  1586. """Execute CreateMultipartUpload S3 API."""
  1587. if not headers.get("Content-Type"):
  1588. headers["Content-Type"] = "application/octet-stream"
  1589. response = self._execute(
  1590. "POST",
  1591. bucket_name,
  1592. object_name,
  1593. headers=headers,
  1594. query_params={"uploads": ""},
  1595. )
  1596. element = ET.fromstring(response.data.decode())
  1597. return cast(str, findtext(element, "UploadId", True))
  1598. def _put_object(
  1599. self,
  1600. bucket_name: str,
  1601. object_name: str,
  1602. data: bytes,
  1603. headers: Optional[DictType] = None,
  1604. query_params: Optional[DictType] = None,
  1605. ) -> ObjectWriteResult:
  1606. """Execute PutObject S3 API."""
  1607. response = self._execute(
  1608. "PUT",
  1609. bucket_name,
  1610. object_name,
  1611. body=data,
  1612. headers=headers,
  1613. query_params=query_params,
  1614. no_body_trace=True,
  1615. )
  1616. return ObjectWriteResult(
  1617. bucket_name,
  1618. object_name,
  1619. response.headers.get("x-amz-version-id"),
  1620. response.headers.get("etag", "").replace('"', ""),
  1621. response.headers,
  1622. )
  1623. def _upload_part(
  1624. self,
  1625. bucket_name: str,
  1626. object_name: str,
  1627. data: bytes,
  1628. headers: Optional[DictType],
  1629. upload_id: str,
  1630. part_number: int,
  1631. ) -> str:
  1632. """Execute UploadPart S3 API."""
  1633. result = self._put_object(
  1634. bucket_name,
  1635. object_name,
  1636. data,
  1637. headers,
  1638. query_params={
  1639. "partNumber": str(part_number),
  1640. "uploadId": upload_id,
  1641. },
  1642. )
  1643. return cast(str, result.etag)
  1644. def _upload_part_task(self, args):
  1645. """Upload_part task for ThreadPool."""
  1646. return args[5], self._upload_part(*args)
  1647. def put_object(
  1648. self,
  1649. bucket_name: str,
  1650. object_name: str,
  1651. data: BinaryIO,
  1652. length: int,
  1653. content_type: str = "application/octet-stream",
  1654. metadata: Optional[DictType] = None,
  1655. sse: Optional[Sse] = None,
  1656. progress: Optional[ProgressType] = None,
  1657. part_size: int = 0,
  1658. num_parallel_uploads: int = 3,
  1659. tags: Optional[Tags] = None,
  1660. retention: Optional[Retention] = None,
  1661. legal_hold: bool = False,
  1662. write_offset: Optional[int] = None,
  1663. ) -> ObjectWriteResult:
  1664. """
  1665. Uploads data from a stream to an object in a bucket.
  1666. :param bucket_name: Name of the bucket.
  1667. :param object_name: Object name in the bucket.
  1668. :param data: An object having callable read() returning bytes object.
  1669. :param length: Data size; -1 for unknown size and set valid part_size.
  1670. :param content_type: Content type of the object.
  1671. :param metadata: Any additional metadata to be uploaded along
  1672. with your PUT request.
  1673. :param sse: Server-side encryption.
  1674. :param progress: A progress object;
  1675. :param part_size: Multipart part size.
  1676. :param num_parallel_uploads: Number of parallel uploads.
  1677. :param tags: :class:`Tags` for the object.
  1678. :param retention: :class:`Retention` configuration object.
  1679. :param legal_hold: Flag to set legal hold for the object.
  1680. :param write_offset: Offset byte for appending data to existing object.
  1681. :return: :class:`ObjectWriteResult` object.
  1682. Example::
  1683. # Upload data.
  1684. result = client.put_object(
  1685. "my-bucket", "my-object", io.BytesIO(b"hello"), 5,
  1686. )
  1687. # Upload data with metadata.
  1688. result = client.put_object(
  1689. "my-bucket", "my-object", io.BytesIO(b"hello"), 5,
  1690. metadata={"My-Project": "one"},
  1691. )
  1692. # Upload data with tags, retention and legal-hold.
  1693. date = datetime.utcnow().replace(
  1694. hour=0, minute=0, second=0, microsecond=0,
  1695. ) + timedelta(days=30)
  1696. tags = Tags(for_object=True)
  1697. tags["User"] = "jsmith"
  1698. result = client.put_object(
  1699. "my-bucket", "my-object", io.BytesIO(b"hello"), 5,
  1700. tags=tags,
  1701. retention=Retention(GOVERNANCE, date),
  1702. legal_hold=True,
  1703. )
  1704. """
  1705. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  1706. check_object_name(object_name)
  1707. check_sse(sse)
  1708. if tags is not None and not isinstance(tags, Tags):
  1709. raise ValueError("tags must be Tags type")
  1710. if retention is not None and not isinstance(retention, Retention):
  1711. raise ValueError("retention must be Retention type")
  1712. if not callable(getattr(data, "read")):
  1713. raise ValueError("input data must have callable read()")
  1714. if write_offset is not None:
  1715. if write_offset < 0:
  1716. raise ValueError("write offset should not be negative")
  1717. if length < 0:
  1718. raise ValueError("length must be provided for write offset")
  1719. part_size = length if length > MIN_PART_SIZE else MIN_PART_SIZE
  1720. part_size, part_count = get_part_info(length, part_size)
  1721. if progress:
  1722. # Set progress bar length and object name before upload
  1723. progress.set_meta(object_name=object_name, total_length=length)
  1724. headers = genheaders(metadata, sse, tags, retention, legal_hold)
  1725. headers["Content-Type"] = content_type or "application/octet-stream"
  1726. if write_offset:
  1727. headers["x-amz-write-offset-bytes"] = str(write_offset)
  1728. object_size = length
  1729. uploaded_size = 0
  1730. part_number = 0
  1731. one_byte = b""
  1732. stop = False
  1733. upload_id = None
  1734. parts: list[Part] = []
  1735. pool: Optional[ThreadPool] = None
  1736. try:
  1737. while not stop:
  1738. part_number += 1
  1739. if part_count > 0:
  1740. if part_number == part_count:
  1741. part_size = object_size - uploaded_size
  1742. stop = True
  1743. part_data = read_part_data(
  1744. data, part_size, progress=progress,
  1745. )
  1746. if len(part_data) != part_size:
  1747. raise IOError(
  1748. f"stream having not enough data;"
  1749. f"expected: {part_size}, "
  1750. f"got: {len(part_data)} bytes"
  1751. )
  1752. else:
  1753. part_data = read_part_data(
  1754. data, part_size + 1, one_byte, progress=progress,
  1755. )
  1756. # If part_data_size is less or equal to part_size,
  1757. # then we have reached last part.
  1758. if len(part_data) <= part_size:
  1759. part_count = part_number
  1760. stop = True
  1761. else:
  1762. one_byte = part_data[-1:]
  1763. part_data = part_data[:-1]
  1764. uploaded_size += len(part_data)
  1765. if part_count == 1:
  1766. return self._put_object(
  1767. bucket_name, object_name, part_data, headers,
  1768. )
  1769. if not upload_id:
  1770. upload_id = self._create_multipart_upload(
  1771. bucket_name, object_name, headers,
  1772. )
  1773. if num_parallel_uploads and num_parallel_uploads > 1:
  1774. pool = ThreadPool(num_parallel_uploads)
  1775. pool.start_parallel()
  1776. args = (
  1777. bucket_name,
  1778. object_name,
  1779. part_data,
  1780. (
  1781. cast(DictType, sse.headers())
  1782. if isinstance(sse, SseCustomerKey) else None
  1783. ),
  1784. upload_id,
  1785. part_number,
  1786. )
  1787. if num_parallel_uploads > 1:
  1788. cast(ThreadPool, pool).add_task(
  1789. self._upload_part_task, args,
  1790. )
  1791. else:
  1792. etag = self._upload_part(*args)
  1793. parts.append(Part(part_number, etag))
  1794. if pool:
  1795. result = pool.result()
  1796. parts = [Part(0, "")] * part_count
  1797. while not result.empty():
  1798. part_number, etag = result.get()
  1799. parts[part_number - 1] = Part(part_number, etag)
  1800. upload_result = self._complete_multipart_upload(
  1801. bucket_name, object_name, cast(str, upload_id), parts,
  1802. sse if isinstance(sse, SseCustomerKey) else None,
  1803. )
  1804. return ObjectWriteResult(
  1805. cast(str, upload_result.bucket_name),
  1806. cast(str, upload_result.object_name),
  1807. upload_result.version_id,
  1808. upload_result.etag,
  1809. upload_result.http_headers,
  1810. location=upload_result.location,
  1811. )
  1812. except Exception as exc:
  1813. if upload_id:
  1814. self._abort_multipart_upload(
  1815. bucket_name, object_name, upload_id,
  1816. )
  1817. raise exc
  1818. def append_object(
  1819. self,
  1820. bucket_name: str,
  1821. object_name: str,
  1822. data: BinaryIO,
  1823. length: int,
  1824. chunk_size: Optional[int] = None,
  1825. progress: Optional[ProgressType] = None,
  1826. extra_headers: Optional[DictType] = None,
  1827. ) -> ObjectWriteResult:
  1828. """
  1829. Appends from a stream to existing object in a bucket.
  1830. :param bucket_name: Name of the bucket.
  1831. :param object_name: Object name in the bucket.
  1832. :param data: An object having callable read() returning bytes object.
  1833. :param length: Data size; -1 for unknown size.
  1834. :param chunk_size: Chunk size to optimize uploads.
  1835. :return: :class:`ObjectWriteResult` object.
  1836. Example::
  1837. # Append data.
  1838. result = client.append_object(
  1839. "my-bucket", "my-object", io.BytesIO(b"world"), 5,
  1840. )
  1841. print(f"appended {result.object_name} object; etag: {result.etag}")
  1842. # Append data in chunks.
  1843. data = urlopen(
  1844. "https://www.kernel.org/pub/linux/kernel/v6.x/"
  1845. "linux-6.13.12.tar.xz",
  1846. )
  1847. result = client.append_object(
  1848. "my-bucket", "my-object", data, 148611164, 5*1024*1024,
  1849. )
  1850. print(f"appended {result.object_name} object; etag: {result.etag}")
  1851. # Append unknown sized data.
  1852. data = urlopen(
  1853. "https://www.kernel.org/pub/linux/kernel/v6.x/"
  1854. "linux-6.14.3.tar.xz",
  1855. )
  1856. result = client.append_object(
  1857. "my-bucket", "my-object", data, 149426584, 5*1024*1024,
  1858. )
  1859. print(f"appended {result.object_name} object; etag: {result.etag}")
  1860. """
  1861. if length == 0:
  1862. raise ValueError("length should not be zero")
  1863. if chunk_size is not None:
  1864. if chunk_size < MIN_PART_SIZE:
  1865. raise ValueError("chunk size must be minimum of 5 MiB")
  1866. if chunk_size > MAX_PART_SIZE:
  1867. raise ValueError("chunk size must be less than 5 GiB")
  1868. else:
  1869. chunk_size = length if length > MIN_PART_SIZE else MIN_PART_SIZE
  1870. chunk_count = -1
  1871. if length > 0:
  1872. chunk_count = int(length / chunk_size)
  1873. if (chunk_count * chunk_size) < length:
  1874. chunk_count += 1
  1875. chunk_count = chunk_count or 1
  1876. object_size = length
  1877. uploaded_size = 0
  1878. chunk_number = 0
  1879. one_byte = b""
  1880. stop = False
  1881. stat = self.stat_object(bucket_name, object_name)
  1882. write_offset = cast(int, stat.size)
  1883. while not stop:
  1884. chunk_number += 1
  1885. if chunk_count > 0:
  1886. if chunk_number == chunk_count:
  1887. chunk_size = object_size - uploaded_size
  1888. stop = True
  1889. chunk_data = read_part_data(
  1890. data, chunk_size, progress=progress,
  1891. )
  1892. if len(chunk_data) != chunk_size:
  1893. raise IOError(
  1894. f"stream having not enough data;"
  1895. f"expected: {chunk_size}, "
  1896. f"got: {len(chunk_data)} bytes"
  1897. )
  1898. else:
  1899. chunk_data = read_part_data(
  1900. data, chunk_size + 1, one_byte, progress=progress,
  1901. )
  1902. # If chunk_data_size is less or equal to chunk_size,
  1903. # then we have reached last chunk.
  1904. if len(chunk_data) <= chunk_size:
  1905. chunk_count = chunk_number
  1906. stop = True
  1907. else:
  1908. one_byte = chunk_data[-1:]
  1909. chunk_data = chunk_data[:-1]
  1910. uploaded_size += len(chunk_data)
  1911. headers = extra_headers or {}
  1912. headers["x-amz-write-offset-bytes"] = str(write_offset)
  1913. upload_result = self._put_object(
  1914. bucket_name, object_name, chunk_data, headers=headers,
  1915. )
  1916. write_offset += len(chunk_data)
  1917. return ObjectWriteResult(
  1918. cast(str, upload_result.bucket_name),
  1919. cast(str, upload_result.object_name),
  1920. upload_result.version_id,
  1921. upload_result.etag,
  1922. upload_result.http_headers,
  1923. location=upload_result.location,
  1924. )
  1925. def list_objects(
  1926. self,
  1927. bucket_name: str,
  1928. prefix: Optional[str] = None,
  1929. recursive: bool = False,
  1930. start_after: Optional[str] = None,
  1931. include_user_meta: bool = False,
  1932. include_version: bool = False,
  1933. use_api_v1: bool = False,
  1934. use_url_encoding_type: bool = True,
  1935. fetch_owner: bool = False,
  1936. extra_headers: Optional[DictType] = None,
  1937. extra_query_params: Optional[DictType] = None,
  1938. ):
  1939. """
  1940. Lists object information of a bucket.
  1941. :param bucket_name: Name of the bucket.
  1942. :param prefix: Object name starts with prefix.
  1943. :param recursive: List recursively than directory structure emulation.
  1944. :param start_after: List objects after this key name.
  1945. :param include_user_meta: MinIO specific flag to control to include
  1946. user metadata.
  1947. :param include_version: Flag to control whether include object
  1948. versions.
  1949. :param use_api_v1: Flag to control to use ListObjectV1 S3 API or not.
  1950. :param use_url_encoding_type: Flag to control whether URL encoding type
  1951. to be used or not.
  1952. :param extra_headers: Extra HTTP headers for advanced usage.
  1953. :param extra_query_params: Extra query parameters for advanced usage.
  1954. :return: Iterator of :class:`Object <Object>`.
  1955. Example::
  1956. # List objects information.
  1957. objects = client.list_objects("my-bucket")
  1958. for obj in objects:
  1959. print(obj)
  1960. # List objects information whose names starts with "my/prefix/".
  1961. objects = client.list_objects("my-bucket", prefix="my/prefix/")
  1962. for obj in objects:
  1963. print(obj)
  1964. # List objects information recursively.
  1965. objects = client.list_objects("my-bucket", recursive=True)
  1966. for obj in objects:
  1967. print(obj)
  1968. # List objects information recursively whose names starts with
  1969. # "my/prefix/".
  1970. objects = client.list_objects(
  1971. "my-bucket", prefix="my/prefix/", recursive=True,
  1972. )
  1973. for obj in objects:
  1974. print(obj)
  1975. # List objects information recursively after object name
  1976. # "my/prefix/world/1".
  1977. objects = client.list_objects(
  1978. "my-bucket", recursive=True, start_after="my/prefix/world/1",
  1979. )
  1980. for obj in objects:
  1981. print(obj)
  1982. """
  1983. return self._list_objects(
  1984. bucket_name,
  1985. delimiter=None if recursive else "/",
  1986. include_user_meta=include_user_meta,
  1987. prefix=prefix,
  1988. start_after=start_after,
  1989. use_api_v1=use_api_v1,
  1990. include_version=include_version,
  1991. encoding_type="url" if use_url_encoding_type else None,
  1992. fetch_owner=fetch_owner,
  1993. extra_headers=extra_headers,
  1994. extra_query_params=extra_query_params,
  1995. )
  1996. def stat_object(
  1997. self,
  1998. bucket_name: str,
  1999. object_name: str,
  2000. ssec: Optional[SseCustomerKey] = None,
  2001. version_id: Optional[str] = None,
  2002. extra_headers: Optional[DictType] = None,
  2003. extra_query_params: Optional[DictType] = None,
  2004. ) -> Object:
  2005. """
  2006. Get object information and metadata of an object.
  2007. :param bucket_name: Name of the bucket.
  2008. :param object_name: Object name in the bucket.
  2009. :param ssec: Server-side encryption customer key.
  2010. :param version_id: Version ID of the object.
  2011. :param extra_headers: Extra HTTP headers for advanced usage.
  2012. :param extra_query_params: Extra query parameters for advanced usage.
  2013. :return: :class:`Object <Object>`.
  2014. Example::
  2015. # Get object information.
  2016. result = client.stat_object("my-bucket", "my-object")
  2017. # Get object information of version-ID.
  2018. result = client.stat_object(
  2019. "my-bucket", "my-object",
  2020. version_id="dfbd25b3-abec-4184-a4e8-5a35a5c1174d",
  2021. )
  2022. # Get SSE-C encrypted object information.
  2023. result = client.stat_object(
  2024. "my-bucket", "my-object",
  2025. ssec=SseCustomerKey(b"32byteslongsecretkeymustprovided"),
  2026. )
  2027. """
  2028. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2029. check_object_name(object_name)
  2030. check_ssec(ssec)
  2031. headers = cast(DictType, ssec.headers() if ssec else {})
  2032. if extra_headers:
  2033. headers.update(extra_headers)
  2034. query_params = extra_query_params or {}
  2035. query_params.update({"versionId": version_id} if version_id else {})
  2036. response = self._execute(
  2037. "HEAD",
  2038. bucket_name,
  2039. object_name,
  2040. headers=headers,
  2041. query_params=query_params,
  2042. )
  2043. value = response.headers.get("last-modified")
  2044. if value is not None:
  2045. last_modified = time.from_http_header(value)
  2046. else:
  2047. last_modified = None
  2048. return Object(
  2049. bucket_name,
  2050. object_name,
  2051. last_modified=last_modified,
  2052. etag=response.headers.get("etag", "").replace('"', ""),
  2053. size=int(response.headers.get("content-length", "0")),
  2054. content_type=response.headers.get("content-type"),
  2055. metadata=response.headers,
  2056. version_id=response.headers.get("x-amz-version-id"),
  2057. )
  2058. def remove_object(
  2059. self,
  2060. bucket_name: str,
  2061. object_name: str,
  2062. version_id: Optional[str] = None
  2063. ):
  2064. """
  2065. Remove an object.
  2066. :param bucket_name: Name of the bucket.
  2067. :param object_name: Object name in the bucket.
  2068. :param version_id: Version ID of the object.
  2069. Example::
  2070. # Remove object.
  2071. client.remove_object("my-bucket", "my-object")
  2072. # Remove version of an object.
  2073. client.remove_object(
  2074. "my-bucket", "my-object",
  2075. version_id="dfbd25b3-abec-4184-a4e8-5a35a5c1174d",
  2076. )
  2077. """
  2078. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2079. check_object_name(object_name)
  2080. self._execute(
  2081. "DELETE",
  2082. bucket_name,
  2083. object_name,
  2084. query_params={"versionId": version_id} if version_id else None,
  2085. )
  2086. def _delete_objects(
  2087. self,
  2088. bucket_name: str,
  2089. delete_object_list: list[DeleteObject],
  2090. quiet: bool = False,
  2091. bypass_governance_mode: bool = False,
  2092. ) -> DeleteResult:
  2093. """
  2094. Delete multiple objects.
  2095. :param bucket_name: Name of the bucket.
  2096. :param delete_object_list: List of maximum 1000
  2097. :class:`DeleteObject <DeleteObject>` object.
  2098. :param quiet: quiet flag.
  2099. :param bypass_governance_mode: Bypass Governance retention mode.
  2100. :return: :class:`DeleteResult <DeleteResult>` object.
  2101. """
  2102. body = marshal(DeleteRequest(delete_object_list, quiet=quiet))
  2103. headers: DictType = {
  2104. "Content-MD5": cast(str, md5sum_hash(body)),
  2105. }
  2106. if bypass_governance_mode:
  2107. headers["x-amz-bypass-governance-retention"] = "true"
  2108. response = self._execute(
  2109. "POST",
  2110. bucket_name,
  2111. body=body,
  2112. headers=headers,
  2113. query_params={"delete": ""},
  2114. )
  2115. element = ET.fromstring(response.data.decode())
  2116. return (
  2117. DeleteResult([], [DeleteError.fromxml(element)])
  2118. if element.tag.endswith("Error")
  2119. else unmarshal(DeleteResult, response.data.decode())
  2120. )
  2121. def remove_objects(
  2122. self,
  2123. bucket_name: str,
  2124. delete_object_list: Iterable[DeleteObject],
  2125. bypass_governance_mode: bool = False,
  2126. ) -> Iterator[DeleteError]:
  2127. """
  2128. Remove multiple objects.
  2129. :param bucket_name: Name of the bucket.
  2130. :param delete_object_list: An iterable containing
  2131. :class:`DeleteObject <DeleteObject>` object.
  2132. :param bypass_governance_mode: Bypass Governance retention mode.
  2133. :return: An iterator containing :class:`DeleteError <DeleteError>`
  2134. object.
  2135. Example::
  2136. # Remove list of objects.
  2137. errors = client.remove_objects(
  2138. "my-bucket",
  2139. [
  2140. DeleteObject("my-object1"),
  2141. DeleteObject("my-object2"),
  2142. DeleteObject(
  2143. "my-object3", "13f88b18-8dcd-4c83-88f2-8631fdb6250c",
  2144. ),
  2145. ],
  2146. )
  2147. for error in errors:
  2148. print("error occurred when deleting object", error)
  2149. # Remove a prefix recursively.
  2150. delete_object_list = list(
  2151. map(
  2152. lambda x: DeleteObject(x.object_name),
  2153. client.list_objects(
  2154. "my-bucket",
  2155. "my/prefix/",
  2156. recursive=True,
  2157. ),
  2158. )
  2159. )
  2160. errors = client.remove_objects("my-bucket", delete_object_list)
  2161. for error in errors:
  2162. print("error occurred when deleting object", error)
  2163. """
  2164. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2165. # turn list like objects into an iterator.
  2166. delete_object_list = itertools.chain(delete_object_list)
  2167. while True:
  2168. # get 1000 entries or whatever available.
  2169. objects = [
  2170. delete_object for _, delete_object in zip(
  2171. range(1000), delete_object_list,
  2172. )
  2173. ]
  2174. if not objects:
  2175. break
  2176. result = self._delete_objects(
  2177. bucket_name,
  2178. objects,
  2179. quiet=True,
  2180. bypass_governance_mode=bypass_governance_mode,
  2181. )
  2182. for error in result.error_list:
  2183. # AWS S3 returns "NoSuchVersion" error when
  2184. # version doesn't exist ignore this error
  2185. # yield all errors otherwise
  2186. if error.code != "NoSuchVersion":
  2187. yield error
  2188. def get_presigned_url(
  2189. self,
  2190. method: str,
  2191. bucket_name: str,
  2192. object_name: str,
  2193. expires: timedelta = timedelta(days=7),
  2194. response_headers: Optional[DictType] = None,
  2195. request_date: Optional[datetime] = None,
  2196. version_id: Optional[str] = None,
  2197. extra_query_params: Optional[DictType] = None,
  2198. ) -> str:
  2199. """
  2200. Get presigned URL of an object for HTTP method, expiry time and custom
  2201. request parameters.
  2202. :param method: HTTP method.
  2203. :param bucket_name: Name of the bucket.
  2204. :param object_name: Object name in the bucket.
  2205. :param expires: Expiry in seconds; defaults to 7 days.
  2206. :param response_headers: Optional response_headers argument to
  2207. specify response fields like date, size,
  2208. type of file, data about server, etc.
  2209. :param request_date: Optional request_date argument to
  2210. specify a different request date. Default is
  2211. current date.
  2212. :param version_id: Version ID of the object.
  2213. :param extra_query_params: Extra query parameters for advanced usage.
  2214. :return: URL string.
  2215. Example::
  2216. # Get presigned URL string to delete 'my-object' in
  2217. # 'my-bucket' with one day expiry.
  2218. url = client.get_presigned_url(
  2219. "DELETE",
  2220. "my-bucket",
  2221. "my-object",
  2222. expires=timedelta(days=1),
  2223. )
  2224. print(url)
  2225. """
  2226. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2227. check_object_name(object_name)
  2228. if expires.total_seconds() < 1 or expires.total_seconds() > 604800:
  2229. raise ValueError("expires must be between 1 second to 7 days")
  2230. region = self._get_region(bucket_name)
  2231. query_params = extra_query_params or {}
  2232. query_params.update({"versionId": version_id} if version_id else {})
  2233. query_params.update(response_headers or {})
  2234. creds = self._provider.retrieve() if self._provider else None
  2235. if creds and creds.session_token:
  2236. query_params["X-Amz-Security-Token"] = creds.session_token
  2237. url = self._base_url.build(
  2238. method=method,
  2239. region=region,
  2240. bucket_name=bucket_name,
  2241. object_name=object_name,
  2242. query_params=query_params,
  2243. )
  2244. if creds:
  2245. url = presign_v4(
  2246. method=method,
  2247. url=url,
  2248. region=region,
  2249. credentials=creds,
  2250. date=request_date or time.utcnow(),
  2251. expires=int(expires.total_seconds()),
  2252. )
  2253. return urlunsplit(url)
  2254. def presigned_get_object(
  2255. self,
  2256. bucket_name: str,
  2257. object_name: str,
  2258. expires: timedelta = timedelta(days=7),
  2259. response_headers: Optional[DictType] = None,
  2260. request_date: Optional[datetime] = None,
  2261. version_id: Optional[str] = None,
  2262. extra_query_params: Optional[DictType] = None,
  2263. ) -> str:
  2264. """
  2265. Get presigned URL of an object to download its data with expiry time
  2266. and custom request parameters.
  2267. :param bucket_name: Name of the bucket.
  2268. :param object_name: Object name in the bucket.
  2269. :param expires: Expiry in seconds; defaults to 7 days.
  2270. :param response_headers: Optional response_headers argument to
  2271. specify response fields like date, size,
  2272. type of file, data about server, etc.
  2273. :param request_date: Optional request_date argument to
  2274. specify a different request date. Default is
  2275. current date.
  2276. :param version_id: Version ID of the object.
  2277. :param extra_query_params: Extra query parameters for advanced usage.
  2278. :return: URL string.
  2279. Example::
  2280. # Get presigned URL string to download 'my-object' in
  2281. # 'my-bucket' with default expiry (i.e. 7 days).
  2282. url = client.presigned_get_object("my-bucket", "my-object")
  2283. print(url)
  2284. # Get presigned URL string to download 'my-object' in
  2285. # 'my-bucket' with two hours expiry.
  2286. url = client.presigned_get_object(
  2287. "my-bucket", "my-object", expires=timedelta(hours=2),
  2288. )
  2289. print(url)
  2290. """
  2291. return self.get_presigned_url(
  2292. "GET",
  2293. bucket_name,
  2294. object_name,
  2295. expires,
  2296. response_headers=response_headers,
  2297. request_date=request_date,
  2298. version_id=version_id,
  2299. extra_query_params=extra_query_params,
  2300. )
  2301. def presigned_put_object(
  2302. self,
  2303. bucket_name: str,
  2304. object_name: str,
  2305. expires: timedelta = timedelta(days=7),
  2306. ) -> str:
  2307. """
  2308. Get presigned URL of an object to upload data with expiry time and
  2309. custom request parameters.
  2310. :param bucket_name: Name of the bucket.
  2311. :param object_name: Object name in the bucket.
  2312. :param expires: Expiry in seconds; defaults to 7 days.
  2313. :return: URL string.
  2314. Example::
  2315. # Get presigned URL string to upload data to 'my-object' in
  2316. # 'my-bucket' with default expiry (i.e. 7 days).
  2317. url = client.presigned_put_object("my-bucket", "my-object")
  2318. print(url)
  2319. # Get presigned URL string to upload data to 'my-object' in
  2320. # 'my-bucket' with two hours expiry.
  2321. url = client.presigned_put_object(
  2322. "my-bucket", "my-object", expires=timedelta(hours=2),
  2323. )
  2324. print(url)
  2325. """
  2326. return self.get_presigned_url(
  2327. "PUT", bucket_name, object_name, expires,
  2328. )
  2329. def presigned_post_policy(self, policy: PostPolicy) -> dict[str, str]:
  2330. """
  2331. Get form-data of PostPolicy of an object to upload its data using POST
  2332. method.
  2333. :param policy: :class:`PostPolicy <PostPolicy>`.
  2334. :return: :dict: contains form-data.
  2335. Example::
  2336. policy = PostPolicy(
  2337. "my-bucket", datetime.utcnow() + timedelta(days=10),
  2338. )
  2339. policy.add_starts_with_condition("key", "my/object/prefix/")
  2340. policy.add_content_length_range_condition(
  2341. 1*1024*1024, 10*1024*1024,
  2342. )
  2343. form_data = client.presigned_post_policy(policy)
  2344. """
  2345. if not isinstance(policy, PostPolicy):
  2346. raise ValueError("policy must be PostPolicy type")
  2347. if not self._provider:
  2348. raise ValueError(
  2349. "anonymous access does not require presigned post form-data",
  2350. )
  2351. check_bucket_name(
  2352. policy.bucket_name, s3_check=self._base_url.is_aws_host)
  2353. return policy.form_data(
  2354. self._provider.retrieve(),
  2355. self._get_region(policy.bucket_name),
  2356. )
  2357. def delete_bucket_replication(self, bucket_name: str):
  2358. """
  2359. Delete replication configuration of a bucket.
  2360. :param bucket_name: Name of the bucket.
  2361. Example::
  2362. client.delete_bucket_replication("my-bucket")
  2363. """
  2364. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2365. self._execute("DELETE", bucket_name, query_params={"replication": ""})
  2366. def get_bucket_replication(
  2367. self,
  2368. bucket_name: str,
  2369. ) -> Optional[ReplicationConfig]:
  2370. """
  2371. Get bucket replication configuration of a bucket.
  2372. :param bucket_name: Name of the bucket.
  2373. :return: :class:`ReplicationConfig <ReplicationConfig>` object.
  2374. Example::
  2375. config = client.get_bucket_replication("my-bucket")
  2376. """
  2377. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2378. try:
  2379. response = self._execute(
  2380. "GET", bucket_name, query_params={"replication": ""},
  2381. )
  2382. return unmarshal(ReplicationConfig, response.data.decode())
  2383. except S3Error as exc:
  2384. if exc.code != "ReplicationConfigurationNotFoundError":
  2385. raise
  2386. return None
  2387. def set_bucket_replication(
  2388. self,
  2389. bucket_name: str,
  2390. config: ReplicationConfig,
  2391. ):
  2392. """
  2393. Set bucket replication configuration to a bucket.
  2394. :param bucket_name: Name of the bucket.
  2395. :param config: :class:`ReplicationConfig <ReplicationConfig>` object.
  2396. Example::
  2397. config = ReplicationConfig(
  2398. "REPLACE-WITH-ACTUAL-ROLE",
  2399. [
  2400. Rule(
  2401. Destination(
  2402. "REPLACE-WITH-ACTUAL-DESTINATION-BUCKET-ARN",
  2403. ),
  2404. ENABLED,
  2405. delete_marker_replication=DeleteMarkerReplication(
  2406. DISABLED,
  2407. ),
  2408. rule_filter=Filter(
  2409. AndOperator(
  2410. "TaxDocs",
  2411. {"key1": "value1", "key2": "value2"},
  2412. ),
  2413. ),
  2414. rule_id="rule1",
  2415. priority=1,
  2416. ),
  2417. ],
  2418. )
  2419. client.set_bucket_replication("my-bucket", config)
  2420. """
  2421. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2422. if not isinstance(config, ReplicationConfig):
  2423. raise ValueError("config must be ReplicationConfig type")
  2424. body = marshal(config)
  2425. self._execute(
  2426. "PUT",
  2427. bucket_name,
  2428. body=body,
  2429. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  2430. query_params={"replication": ""},
  2431. )
  2432. def delete_bucket_lifecycle(self, bucket_name: str):
  2433. """
  2434. Delete notification configuration of a bucket.
  2435. :param bucket_name: Name of the bucket.
  2436. Example::
  2437. client.delete_bucket_lifecycle("my-bucket")
  2438. """
  2439. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2440. self._execute("DELETE", bucket_name, query_params={"lifecycle": ""})
  2441. def get_bucket_lifecycle(
  2442. self,
  2443. bucket_name: str,
  2444. ) -> Optional[LifecycleConfig]:
  2445. """
  2446. Get bucket lifecycle configuration of a bucket.
  2447. :param bucket_name: Name of the bucket.
  2448. :return: :class:`LifecycleConfig <LifecycleConfig>` object.
  2449. Example::
  2450. config = client.get_bucket_lifecycle("my-bucket")
  2451. """
  2452. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2453. try:
  2454. response = self._execute(
  2455. "GET", bucket_name, query_params={"lifecycle": ""},
  2456. )
  2457. return unmarshal(LifecycleConfig, response.data.decode())
  2458. except S3Error as exc:
  2459. if exc.code != "NoSuchLifecycleConfiguration":
  2460. raise
  2461. return None
  2462. def set_bucket_lifecycle(
  2463. self,
  2464. bucket_name: str,
  2465. config: LifecycleConfig,
  2466. ):
  2467. """
  2468. Set bucket lifecycle configuration to a bucket.
  2469. :param bucket_name: Name of the bucket.
  2470. :param config: :class:`LifecycleConfig <LifecycleConfig>` object.
  2471. Example::
  2472. config = LifecycleConfig(
  2473. [
  2474. Rule(
  2475. ENABLED,
  2476. rule_filter=Filter(prefix="documents/"),
  2477. rule_id="rule1",
  2478. transition=Transition(
  2479. days=30, storage_class="GLACIER",
  2480. ),
  2481. ),
  2482. Rule(
  2483. ENABLED,
  2484. rule_filter=Filter(prefix="logs/"),
  2485. rule_id="rule2",
  2486. expiration=Expiration(days=365),
  2487. ),
  2488. ],
  2489. )
  2490. client.set_bucket_lifecycle("my-bucket", config)
  2491. """
  2492. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2493. if not isinstance(config, LifecycleConfig):
  2494. raise ValueError("config must be LifecycleConfig type")
  2495. body = marshal(config)
  2496. self._execute(
  2497. "PUT",
  2498. bucket_name,
  2499. body=body,
  2500. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  2501. query_params={"lifecycle": ""},
  2502. )
  2503. def delete_bucket_tags(self, bucket_name: str):
  2504. """
  2505. Delete tags configuration of a bucket.
  2506. :param bucket_name: Name of the bucket.
  2507. Example::
  2508. client.delete_bucket_tags("my-bucket")
  2509. """
  2510. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2511. self._execute("DELETE", bucket_name, query_params={"tagging": ""})
  2512. def get_bucket_tags(self, bucket_name: str) -> Optional[Tags]:
  2513. """
  2514. Get tags configuration of a bucket.
  2515. :param bucket_name: Name of the bucket.
  2516. :return: :class:`Tags <Tags>` object.
  2517. Example::
  2518. tags = client.get_bucket_tags("my-bucket")
  2519. """
  2520. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2521. try:
  2522. response = self._execute(
  2523. "GET", bucket_name, query_params={"tagging": ""},
  2524. )
  2525. tagging = unmarshal(Tagging, response.data.decode())
  2526. return tagging.tags
  2527. except S3Error as exc:
  2528. if exc.code != "NoSuchTagSet":
  2529. raise
  2530. return None
  2531. def set_bucket_tags(self, bucket_name: str, tags: Tags):
  2532. """
  2533. Set tags configuration to a bucket.
  2534. :param bucket_name: Name of the bucket.
  2535. :param tags: :class:`Tags <Tags>` object.
  2536. Example::
  2537. tags = Tags.new_bucket_tags()
  2538. tags["Project"] = "Project One"
  2539. tags["User"] = "jsmith"
  2540. client.set_bucket_tags("my-bucket", tags)
  2541. """
  2542. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2543. if not isinstance(tags, Tags):
  2544. raise ValueError("tags must be Tags type")
  2545. body = marshal(Tagging(tags))
  2546. self._execute(
  2547. "PUT",
  2548. bucket_name,
  2549. body=body,
  2550. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  2551. query_params={"tagging": ""},
  2552. )
  2553. def delete_object_tags(
  2554. self,
  2555. bucket_name: str,
  2556. object_name: str,
  2557. version_id: Optional[str] = None,
  2558. ):
  2559. """
  2560. Delete tags configuration of an object.
  2561. :param bucket_name: Name of the bucket.
  2562. :param object_name: Object name in the bucket.
  2563. :param version_id: Version ID of the Object.
  2564. Example::
  2565. client.delete_object_tags("my-bucket", "my-object")
  2566. """
  2567. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2568. check_object_name(object_name)
  2569. query_params = {"versionId": version_id} if version_id else {}
  2570. query_params["tagging"] = ""
  2571. self._execute(
  2572. "DELETE",
  2573. bucket_name,
  2574. object_name=object_name,
  2575. query_params=cast(DictType, query_params),
  2576. )
  2577. def get_object_tags(
  2578. self,
  2579. bucket_name: str,
  2580. object_name: str,
  2581. version_id: Optional[str] = None,
  2582. ) -> Optional[Tags]:
  2583. """
  2584. Get tags configuration of a object.
  2585. :param bucket_name: Name of the bucket.
  2586. :param object_name: Object name in the bucket.
  2587. :param version_id: Version ID of the Object.
  2588. :return: :class:`Tags <Tags>` object.
  2589. Example::
  2590. tags = client.get_object_tags("my-bucket", "my-object")
  2591. """
  2592. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2593. check_object_name(object_name)
  2594. query_params = {"versionId": version_id} if version_id else {}
  2595. query_params["tagging"] = ""
  2596. try:
  2597. response = self._execute(
  2598. "GET",
  2599. bucket_name,
  2600. object_name=object_name,
  2601. query_params=cast(DictType, query_params),
  2602. )
  2603. tagging = unmarshal(Tagging, response.data.decode())
  2604. return tagging.tags
  2605. except S3Error as exc:
  2606. if exc.code != "NoSuchTagSet":
  2607. raise
  2608. return None
  2609. def set_object_tags(
  2610. self,
  2611. bucket_name: str,
  2612. object_name: str,
  2613. tags: Tags,
  2614. version_id: Optional[str] = None,
  2615. ):
  2616. """
  2617. Set tags configuration to an object.
  2618. :param bucket_name: Name of the bucket.
  2619. :param object_name: Object name in the bucket.
  2620. :param version_id: Version ID of the Object.
  2621. :param tags: :class:`Tags <Tags>` object.
  2622. Example::
  2623. tags = Tags.new_object_tags()
  2624. tags["Project"] = "Project One"
  2625. tags["User"] = "jsmith"
  2626. client.set_object_tags("my-bucket", "my-object", tags)
  2627. """
  2628. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2629. check_object_name(object_name)
  2630. if not isinstance(tags, Tags):
  2631. raise ValueError("tags must be Tags type")
  2632. body = marshal(Tagging(tags))
  2633. query_params = {"versionId": version_id} if version_id else {}
  2634. query_params["tagging"] = ""
  2635. self._execute(
  2636. "PUT",
  2637. bucket_name,
  2638. object_name=object_name,
  2639. body=body,
  2640. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  2641. query_params=cast(DictType, query_params),
  2642. )
  2643. def enable_object_legal_hold(
  2644. self,
  2645. bucket_name: str,
  2646. object_name: str,
  2647. version_id: Optional[str] = None,
  2648. ):
  2649. """
  2650. Enable legal hold on an object.
  2651. :param bucket_name: Name of the bucket.
  2652. :param object_name: Object name in the bucket.
  2653. :param version_id: Version ID of the object.
  2654. Example::
  2655. client.enable_object_legal_hold("my-bucket", "my-object")
  2656. """
  2657. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2658. check_object_name(object_name)
  2659. body = marshal(LegalHold(True))
  2660. query_params = {"versionId": version_id} if version_id else {}
  2661. query_params["legal-hold"] = ""
  2662. self._execute(
  2663. "PUT",
  2664. bucket_name,
  2665. object_name=object_name,
  2666. body=body,
  2667. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  2668. query_params=cast(DictType, query_params),
  2669. )
  2670. def disable_object_legal_hold(
  2671. self,
  2672. bucket_name: str,
  2673. object_name: str,
  2674. version_id: Optional[str] = None,
  2675. ):
  2676. """
  2677. Disable legal hold on an object.
  2678. :param bucket_name: Name of the bucket.
  2679. :param object_name: Object name in the bucket.
  2680. :param version_id: Version ID of the object.
  2681. Example::
  2682. client.disable_object_legal_hold("my-bucket", "my-object")
  2683. """
  2684. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2685. check_object_name(object_name)
  2686. body = marshal(LegalHold(False))
  2687. query_params = {"versionId": version_id} if version_id else {}
  2688. query_params["legal-hold"] = ""
  2689. self._execute(
  2690. "PUT",
  2691. bucket_name,
  2692. object_name=object_name,
  2693. body=body,
  2694. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  2695. query_params=cast(DictType, query_params),
  2696. )
  2697. def is_object_legal_hold_enabled(
  2698. self,
  2699. bucket_name: str,
  2700. object_name: str,
  2701. version_id: Optional[str] = None,
  2702. ) -> bool:
  2703. """
  2704. Returns true if legal hold is enabled on an object.
  2705. :param bucket_name: Name of the bucket.
  2706. :param object_name: Object name in the bucket.
  2707. :param version_id: Version ID of the object.
  2708. Example::
  2709. if client.is_object_legal_hold_enabled("my-bucket", "my-object"):
  2710. print("legal hold is enabled on my-object")
  2711. else:
  2712. print("legal hold is not enabled on my-object")
  2713. """
  2714. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2715. check_object_name(object_name)
  2716. query_params = {"versionId": version_id} if version_id else {}
  2717. query_params["legal-hold"] = ""
  2718. try:
  2719. response = self._execute(
  2720. "GET",
  2721. bucket_name,
  2722. object_name=object_name,
  2723. query_params=cast(DictType, query_params),
  2724. )
  2725. legal_hold = unmarshal(LegalHold, response.data.decode())
  2726. return legal_hold.status
  2727. except S3Error as exc:
  2728. if exc.code != "NoSuchObjectLockConfiguration":
  2729. raise
  2730. return False
  2731. def delete_object_lock_config(self, bucket_name: str):
  2732. """
  2733. Delete object-lock configuration of a bucket.
  2734. :param bucket_name: Name of the bucket.
  2735. Example::
  2736. client.delete_object_lock_config("my-bucket")
  2737. """
  2738. self.set_object_lock_config(
  2739. bucket_name, ObjectLockConfig(None, None, None)
  2740. )
  2741. def get_object_lock_config(self, bucket_name: str) -> ObjectLockConfig:
  2742. """
  2743. Get object-lock configuration of a bucket.
  2744. :param bucket_name: Name of the bucket.
  2745. :return: :class:`ObjectLockConfig <ObjectLockConfig>` object.
  2746. Example::
  2747. config = client.get_object_lock_config("my-bucket")
  2748. """
  2749. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2750. response = self._execute(
  2751. "GET", bucket_name, query_params={"object-lock": ""},
  2752. )
  2753. return unmarshal(ObjectLockConfig, response.data.decode())
  2754. def set_object_lock_config(
  2755. self,
  2756. bucket_name: str,
  2757. config: ObjectLockConfig,
  2758. ):
  2759. """
  2760. Set object-lock configuration to a bucket.
  2761. :param bucket_name: Name of the bucket.
  2762. :param config: :class:`ObjectLockConfig <ObjectLockConfig>` object.
  2763. Example::
  2764. config = ObjectLockConfig(GOVERNANCE, 15, DAYS)
  2765. client.set_object_lock_config("my-bucket", config)
  2766. """
  2767. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2768. if not isinstance(config, ObjectLockConfig):
  2769. raise ValueError("config must be ObjectLockConfig type")
  2770. body = marshal(config)
  2771. self._execute(
  2772. "PUT",
  2773. bucket_name,
  2774. body=body,
  2775. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  2776. query_params={"object-lock": ""},
  2777. )
  2778. def get_object_retention(
  2779. self,
  2780. bucket_name: str,
  2781. object_name: str,
  2782. version_id: Optional[str] = None,
  2783. ) -> Optional[Retention]:
  2784. """
  2785. Get retention configuration of an object.
  2786. :param bucket_name: Name of the bucket.
  2787. :param object_name: Object name in the bucket.
  2788. :param version_id: Version ID of the object.
  2789. :return: :class:`Retention <Retention>` object.
  2790. Example::
  2791. config = client.get_object_retention("my-bucket", "my-object")
  2792. """
  2793. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2794. check_object_name(object_name)
  2795. query_params = {"versionId": version_id} if version_id else {}
  2796. query_params["retention"] = ""
  2797. try:
  2798. response = self._execute(
  2799. "GET",
  2800. bucket_name,
  2801. object_name=object_name,
  2802. query_params=cast(DictType, query_params),
  2803. )
  2804. return unmarshal(Retention, response.data.decode())
  2805. except S3Error as exc:
  2806. if exc.code != "NoSuchObjectLockConfiguration":
  2807. raise
  2808. return None
  2809. def set_object_retention(
  2810. self,
  2811. bucket_name: str,
  2812. object_name: str,
  2813. config: Retention,
  2814. version_id: Optional[str] = None,
  2815. ):
  2816. """
  2817. Set retention configuration on an object.
  2818. :param bucket_name: Name of the bucket.
  2819. :param object_name: Object name in the bucket.
  2820. :param version_id: Version ID of the object.
  2821. :param config: :class:`Retention <Retention>` object.
  2822. Example::
  2823. config = Retention(
  2824. GOVERNANCE, datetime.utcnow() + timedelta(days=10),
  2825. )
  2826. client.set_object_retention("my-bucket", "my-object", config)
  2827. """
  2828. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2829. check_object_name(object_name)
  2830. if not isinstance(config, Retention):
  2831. raise ValueError("config must be Retention type")
  2832. body = marshal(config)
  2833. query_params = {"versionId": version_id} if version_id else {}
  2834. query_params["retention"] = ""
  2835. self._execute(
  2836. "PUT",
  2837. bucket_name,
  2838. object_name=object_name,
  2839. body=body,
  2840. headers={"Content-MD5": cast(str, md5sum_hash(body))},
  2841. query_params=cast(DictType, query_params),
  2842. )
  2843. def upload_snowball_objects(
  2844. self,
  2845. bucket_name: str,
  2846. object_list: Iterable[SnowballObject],
  2847. metadata: Optional[DictType] = None,
  2848. sse: Optional[Sse] = None,
  2849. tags: Optional[Tags] = None,
  2850. retention: Optional[Retention] = None,
  2851. legal_hold: bool = False,
  2852. staging_filename: Optional[str] = None,
  2853. compression: bool = False,
  2854. ) -> ObjectWriteResult:
  2855. """
  2856. Uploads multiple objects in a single put call. It is done by creating
  2857. intermediate TAR file optionally compressed which is uploaded to S3
  2858. service.
  2859. :param bucket_name: Name of the bucket.
  2860. :param object_list: An iterable containing
  2861. :class:`SnowballObject <SnowballObject>` object.
  2862. :param metadata: Any additional metadata to be uploaded along
  2863. with your PUT request.
  2864. :param sse: Server-side encryption.
  2865. :param tags: :class:`Tags` for the object.
  2866. :param retention: :class:`Retention` configuration object.
  2867. :param legal_hold: Flag to set legal hold for the object.
  2868. :param staging_filename: A staging filename to create intermediate
  2869. tarball.
  2870. :param compression: Flag to compress TAR ball.
  2871. :return: :class:`ObjectWriteResult` object.
  2872. Example::
  2873. # Upload snowball object.
  2874. result = client.upload_snowball_objects(
  2875. "my-bucket",
  2876. [
  2877. SnowballObject("my-object1", filename="/etc/hostname"),
  2878. SnowballObject(
  2879. "my-object2", data=io.BytesIO("hello"), length=5,
  2880. ),
  2881. SnowballObject(
  2882. "my-object3", data=io.BytesIO("world"), length=5,
  2883. mod_time=datetime.now(),
  2884. ),
  2885. ],
  2886. )
  2887. """
  2888. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2889. object_name = f"snowball.{random()}.tar"
  2890. # turn list like objects into an iterator.
  2891. object_list = itertools.chain(object_list)
  2892. metadata = metadata or {}
  2893. metadata["X-Amz-Meta-Snowball-Auto-Extract"] = "true"
  2894. name = staging_filename
  2895. fileobj = None if name else BytesIO()
  2896. with tarfile.open(
  2897. name=name, mode="w:gz" if compression else "w", fileobj=fileobj,
  2898. ) as tar:
  2899. for obj in object_list:
  2900. if obj.filename:
  2901. tar.add(obj.filename, obj.object_name)
  2902. else:
  2903. info = tarfile.TarInfo(obj.object_name)
  2904. info.size = cast(int, obj.length)
  2905. info.mtime = int(
  2906. time.to_float(obj.mod_time or time.utcnow()),
  2907. )
  2908. tar.addfile(info, obj.data)
  2909. if not name:
  2910. length = cast(BytesIO, fileobj).tell()
  2911. cast(BytesIO, fileobj).seek(0)
  2912. else:
  2913. length = os.stat(name).st_size
  2914. part_size = 0 if length < MIN_PART_SIZE else length
  2915. if name:
  2916. return self.fput_object(
  2917. bucket_name,
  2918. object_name,
  2919. cast(str, staging_filename),
  2920. metadata=metadata,
  2921. sse=sse,
  2922. tags=tags,
  2923. retention=retention,
  2924. legal_hold=legal_hold,
  2925. part_size=part_size,
  2926. )
  2927. return self.put_object(
  2928. bucket_name,
  2929. object_name,
  2930. cast(BinaryIO, fileobj),
  2931. length,
  2932. metadata=cast(Union[DictType, None], metadata),
  2933. sse=sse,
  2934. tags=tags,
  2935. retention=retention,
  2936. legal_hold=legal_hold,
  2937. part_size=part_size,
  2938. )
  2939. def _list_objects(
  2940. self,
  2941. bucket_name: str,
  2942. continuation_token: Optional[str] = None, # listV2 only
  2943. delimiter: Optional[str] = None, # all
  2944. encoding_type: Optional[str] = None, # all
  2945. fetch_owner: Optional[bool] = None, # listV2 only
  2946. include_user_meta: bool = False, # MinIO specific listV2.
  2947. max_keys: Optional[int] = None, # all
  2948. prefix: Optional[str] = None, # all
  2949. start_after: Optional[str] = None,
  2950. # all: v1:marker, versioned:key_marker
  2951. version_id_marker: Optional[str] = None, # versioned
  2952. use_api_v1: bool = False,
  2953. include_version: bool = False,
  2954. extra_headers: Optional[DictType] = None,
  2955. extra_query_params: Optional[DictType] = None,
  2956. ) -> Iterator[Object]:
  2957. """
  2958. List objects optionally including versions.
  2959. Note: Its required to send empty values to delimiter/prefix and 1000 to
  2960. max-keys when not provided for server-side bucket policy evaluation to
  2961. succeed; otherwise AccessDenied error will be returned for such
  2962. policies.
  2963. """
  2964. check_bucket_name(bucket_name, s3_check=self._base_url.is_aws_host)
  2965. if version_id_marker:
  2966. include_version = True
  2967. is_truncated = True
  2968. while is_truncated:
  2969. query = extra_query_params or {}
  2970. if include_version:
  2971. query["versions"] = ""
  2972. elif not use_api_v1:
  2973. query["list-type"] = "2"
  2974. if not include_version and not use_api_v1:
  2975. if continuation_token:
  2976. query["continuation-token"] = continuation_token
  2977. if fetch_owner:
  2978. query["fetch-owner"] = "true"
  2979. if include_user_meta:
  2980. query["metadata"] = "true"
  2981. query["delimiter"] = delimiter or ""
  2982. if encoding_type:
  2983. query["encoding-type"] = encoding_type
  2984. query["max-keys"] = str(max_keys or 1000)
  2985. query["prefix"] = prefix or ""
  2986. if start_after:
  2987. if include_version:
  2988. query["key-marker"] = start_after
  2989. elif use_api_v1:
  2990. query["marker"] = start_after
  2991. else:
  2992. query["start-after"] = start_after
  2993. if version_id_marker:
  2994. query["version-id-marker"] = version_id_marker
  2995. response = self._execute(
  2996. "GET",
  2997. bucket_name,
  2998. query_params=cast(DictType, query),
  2999. headers=extra_headers,
  3000. )
  3001. objects, is_truncated, start_after, version_id_marker = (
  3002. parse_list_objects(response)
  3003. )
  3004. if not include_version:
  3005. version_id_marker = None
  3006. if not use_api_v1:
  3007. continuation_token = start_after
  3008. yield from objects
  3009. def _list_multipart_uploads(
  3010. self,
  3011. bucket_name: str,
  3012. delimiter: Optional[str] = None,
  3013. encoding_type: Optional[str] = None,
  3014. key_marker: Optional[str] = None,
  3015. max_uploads: Optional[int] = None,
  3016. prefix: Optional[str] = None,
  3017. upload_id_marker: Optional[str] = None,
  3018. extra_headers: Optional[DictType] = None,
  3019. extra_query_params: Optional[DictType] = None,
  3020. ) -> ListMultipartUploadsResult:
  3021. """
  3022. Execute ListMultipartUploads S3 API.
  3023. :param bucket_name: Name of the bucket.
  3024. :param delimiter: (Optional) Delimiter on listing.
  3025. :param encoding_type: (Optional) Encoding type.
  3026. :param key_marker: (Optional) Key marker.
  3027. :param max_uploads: (Optional) Maximum upload information to fetch.
  3028. :param prefix: (Optional) Prefix on listing.
  3029. :param upload_id_marker: (Optional) Upload ID marker.
  3030. :param extra_headers: (Optional) Extra headers for advanced usage.
  3031. :param extra_query_params: (Optional) Extra query parameters for
  3032. advanced usage.
  3033. :return:
  3034. :class:`ListMultipartUploadsResult <ListMultipartUploadsResult>`
  3035. object
  3036. """
  3037. query_params = extra_query_params or {}
  3038. query_params.update(
  3039. {
  3040. "uploads": "",
  3041. "delimiter": delimiter or "",
  3042. "max-uploads": str(max_uploads or 1000),
  3043. "prefix": prefix or "",
  3044. "encoding-type": "url",
  3045. },
  3046. )
  3047. if encoding_type:
  3048. query_params["encoding-type"] = encoding_type
  3049. if key_marker:
  3050. query_params["key-marker"] = key_marker
  3051. if upload_id_marker:
  3052. query_params["upload-id-marker"] = upload_id_marker
  3053. response = self._execute(
  3054. "GET",
  3055. bucket_name,
  3056. query_params=cast(DictType, query_params),
  3057. headers=cast(Union[DictType, None], extra_headers),
  3058. )
  3059. return ListMultipartUploadsResult(response)
  3060. def _list_parts(
  3061. self,
  3062. bucket_name: str,
  3063. object_name: str,
  3064. upload_id: str,
  3065. max_parts: Optional[int] = None,
  3066. part_number_marker: Optional[str] = None,
  3067. extra_headers: Optional[DictType] = None,
  3068. extra_query_params: Optional[DictType] = None,
  3069. ) -> ListPartsResult:
  3070. """
  3071. Execute ListParts S3 API.
  3072. :param bucket_name: Name of the bucket.
  3073. :param object_name: Object name in the bucket.
  3074. :param upload_id: Upload ID.
  3075. :param max_parts: (Optional) Maximum parts information to fetch.
  3076. :param part_number_marker: (Optional) Part number marker.
  3077. :param extra_headers: (Optional) Extra headers for advanced usage.
  3078. :param extra_query_params: (Optional) Extra query parameters for
  3079. advanced usage.
  3080. :return: :class:`ListPartsResult <ListPartsResult>` object
  3081. """
  3082. query_params = extra_query_params or {}
  3083. query_params.update(
  3084. {
  3085. "uploadId": upload_id,
  3086. "max-parts": str(max_parts or 1000),
  3087. },
  3088. )
  3089. if part_number_marker:
  3090. query_params["part-number-marker"] = part_number_marker
  3091. response = self._execute(
  3092. "GET",
  3093. bucket_name,
  3094. object_name=object_name,
  3095. query_params=cast(DictType, query_params),
  3096. headers=cast(Union[DictType, None], extra_headers),
  3097. )
  3098. return ListPartsResult(response)