mirror of
https://github.com/diced/zipline.git
synced 2025-04-11 23:31:17 -05:00
fix: malformed s3 multipart uploads (#771)
This commit is contained in:
diced
parent
55684528b8
commit
c2848f19c1
3 changed files with 327 additions and 130 deletions
|
|
@ -21,6 +21,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.726.1",
|
||||
"@aws-sdk/lib-storage": "3.726.1",
|
||||
"@fastify/cookie": "^11.0.2",
|
||||
"@fastify/cors": "^10.0.2",
|
||||
"@fastify/multipart": "^9.0.3",
|
||||
|
|
|
|||
272
pnpm-lock.yaml
generated
272
pnpm-lock.yaml
generated
|
|
@ -11,6 +11,9 @@ importers:
|
|||
'@aws-sdk/client-s3':
|
||||
specifier: 3.726.1
|
||||
version: 3.726.1
|
||||
'@aws-sdk/lib-storage':
|
||||
specifier: 3.726.1
|
||||
version: 3.726.1(@aws-sdk/client-s3@3.726.1)
|
||||
'@fastify/cookie':
|
||||
specifier: ^11.0.2
|
||||
version: 11.0.2
|
||||
|
|
@ -354,6 +357,12 @@ packages:
|
|||
peerDependencies:
|
||||
'@aws-sdk/client-sts': ^3.723.0
|
||||
|
||||
'@aws-sdk/lib-storage@3.726.1':
|
||||
resolution: {integrity: sha512-WuDxSZ8Bfe1N7gn5eXQ02dhlKWCAwW5qQErpJ4CCddXosF+gLxhGkrP9LkaaP0CpA3PxboHyET6HbWAggOWtqA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
peerDependencies:
|
||||
'@aws-sdk/client-s3': ^3.726.1
|
||||
|
||||
'@aws-sdk/middleware-bucket-endpoint@3.726.0':
|
||||
resolution: {integrity: sha512-vpaP80rZqwu0C3ELayIcRIW84/nd1tadeoqllT+N9TDshuEvq4UJ+w47OBHB7RkHFJoc79lXXNYle0fdQdaE/A==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1602,6 +1611,10 @@ packages:
|
|||
resolution: {integrity: sha512-wFExFGK+7r2wYriOqe7RRIBNpvxwiS95ih09+GSLRBdoyK/O1uZA7K7pKesj5CBvwJuSBeXwLyR88WwIAY+DGA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/core@3.2.0':
|
||||
resolution: {integrity: sha512-k17bgQhVZ7YmUvA8at4af1TDpl0NDMBuBKJl8Yg0nrefwmValU+CnA5l/AriVdQNthU/33H3nK71HrLgqOPr1Q==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/credential-provider-imds@4.0.1':
|
||||
resolution: {integrity: sha512-l/qdInaDq1Zpznpmev/+52QomsJNZ3JkTl5yrTl02V6NBgJOQ4LY0SFw/8zsMwj3tLe8vqiIuwF6nxaEwgf6mg==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1630,6 +1643,10 @@ packages:
|
|||
resolution: {integrity: sha512-3aS+fP28urrMW2KTjb6z9iFow6jO8n3MFfineGbndvzGZit3taZhKWtTorf+Gp5RpFDDafeHlhfsGlDCXvUnJA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/fetch-http-handler@5.0.2':
|
||||
resolution: {integrity: sha512-+9Dz8sakS9pe7f2cBocpJXdeVjMopUDLgZs1yWeu7h++WqSbjUYv/JAJwKwXw1HV6gq1jyWjxuyn24E2GhoEcQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/hash-blob-browser@4.0.1':
|
||||
resolution: {integrity: sha512-rkFIrQOKZGS6i1D3gKJ8skJ0RlXqDvb1IyAphksaFOMzkn3v3I1eJ8m7OkLj0jf1McP63rcCEoLlkAn/HjcTRw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1666,6 +1683,10 @@ packages:
|
|||
resolution: {integrity: sha512-cPzGZV7qStHwboFrm6GfrzQE+YDiCzWcTh4+7wKrP/ZQ4gkw+r7qDjV8GjM4N0UYsuUyLfpzLGg5hxsYTU11WA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/middleware-endpoint@4.1.0':
|
||||
resolution: {integrity: sha512-xhLimgNCbCzsUppRTGXWkZywksuTThxaIB0HwbpsVLY5sceac4e1TZ/WKYqufQLaUy+gUSJGNdwD2jo3cXL0iA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/middleware-retry@4.0.6':
|
||||
resolution: {integrity: sha512-s8QzuOQnbdvRymD9Gt9c9zMq10wUQAHQ3z72uirrBHCwZcLTrL5iCOuVTMdka2IXOYhQE890WD5t6G24+F+Qcg==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1674,14 +1695,26 @@ packages:
|
|||
resolution: {integrity: sha512-Sdr5lOagCn5tt+zKsaW+U2/iwr6bI9p08wOkCp6/eL6iMbgdtc2R5Ety66rf87PeohR0ExI84Txz9GYv5ou3iQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/middleware-serde@4.0.3':
|
||||
resolution: {integrity: sha512-rfgDVrgLEVMmMn0BI8O+8OVr6vXzjV7HZj57l0QxslhzbvVfikZbVfBVthjLHqib4BW44QhcIgJpvebHlRaC9A==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/middleware-stack@4.0.1':
|
||||
resolution: {integrity: sha512-dHwDmrtR/ln8UTHpaIavRSzeIk5+YZTBtLnKwDW3G2t6nAupCiQUvNzNoHBpik63fwUaJPtlnMzXbQrNFWssIA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/middleware-stack@4.0.2':
|
||||
resolution: {integrity: sha512-eSPVcuJJGVYrFYu2hEq8g8WWdJav3sdrI4o2c6z/rjnYDd3xH9j9E7deZQCzFn4QvGPouLngH3dQ+QVTxv5bOQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/node-config-provider@4.0.1':
|
||||
resolution: {integrity: sha512-8mRTjvCtVET8+rxvmzRNRR0hH2JjV0DFOmwXPrISmTIJEfnCBugpYYGAsCj8t41qd+RB5gbheSQ/6aKZCQvFLQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/node-config-provider@4.0.2':
|
||||
resolution: {integrity: sha512-WgCkILRZfJwJ4Da92a6t3ozN/zcvYyJGUTmfGbgS/FkCcoCjl7G4FJaCDN1ySdvLvemnQeo25FdkyMSTSwulsw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/node-http-handler@4.0.4':
|
||||
resolution: {integrity: sha512-/mdqabuAT3o/ihBGjL94PUbTSPSRJ0eeVTdgADzow0wRJ0rN4A27EOrtlK56MYiO1fDvlO3jVTCxQtQmK9dZ1g==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1690,6 +1723,10 @@ packages:
|
|||
resolution: {integrity: sha512-o+VRiwC2cgmk/WFV0jaETGOtX16VNPp2bSQEzu0whbReqE1BMqsP2ami2Vi3cbGVdKu1kq9gQkDAGKbt0WOHAQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/property-provider@4.0.2':
|
||||
resolution: {integrity: sha512-wNRoQC1uISOuNc2s4hkOYwYllmiyrvVXWMtq+TysNRVQaHm4yoafYQyjN/goYZS+QbYlPIbb/QRjaUZMuzwQ7A==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/protocol-http@5.0.1':
|
||||
resolution: {integrity: sha512-TE4cpj49jJNB/oHyh/cRVEgNZaoPaxd4vteJNB0yGidOCVR0jCw/hjPVsT8Q8FRmj8Bd3bFZt8Dh7xGCT+xMBQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1710,6 +1747,10 @@ packages:
|
|||
resolution: {integrity: sha512-Ma2XC7VS9aV77+clSFylVUnPZRindhB7BbmYiNOdr+CHt/kZNJoPP0cd3QxCnCFyPXC4eybmyE98phEHkqZ5Jw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/querystring-parser@4.0.2':
|
||||
resolution: {integrity: sha512-v6w8wnmZcVXjfVLjxw8qF7OwESD9wnpjp0Dqry/Pod0/5vcEA3qxCr+BhbOHlxS8O+29eLpT3aagxXGwIoEk7Q==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/service-error-classification@4.0.1':
|
||||
resolution: {integrity: sha512-3JNjBfOWpj/mYfjXJHB4Txc/7E4LVq32bwzE7m28GN79+M1f76XHflUaSUkhOriprPDzev9cX/M+dEB80DNDKA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1718,6 +1759,10 @@ packages:
|
|||
resolution: {integrity: sha512-hC8F6qTBbuHRI/uqDgqqi6J0R4GtEZcgrZPhFQnMhfJs3MnUTGSnR1NSJCJs5VWlMydu0kJz15M640fJlRsIOw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/shared-ini-file-loader@4.0.2':
|
||||
resolution: {integrity: sha512-J9/gTWBGVuFZ01oVA6vdb4DAjf1XbDhK6sLsu3OS9qmLrS6KB5ygpeHiM3miIbj1qgSJ96GYszXFWv6ErJ8QEw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/signature-v4@5.0.1':
|
||||
resolution: {integrity: sha512-nCe6fQ+ppm1bQuw5iKoeJ0MJfz2os7Ic3GBjOkLOPtavbD1ONoyE3ygjBfz2ythFWm4YnRm6OxW+8p/m9uCoIA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1726,6 +1771,10 @@ packages:
|
|||
resolution: {integrity: sha512-DMXYoYeL4QkElr216n1yodTFeATbfb4jwYM9gKn71Rw/FNA1/Sm36tkTSCsZEs7mgpG3OINmkxL9vgVFzyGPaw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/smithy-client@4.2.0':
|
||||
resolution: {integrity: sha512-Qs65/w30pWV7LSFAez9DKy0Koaoh3iHhpcpCCJ4waj/iqwsuSzJna2+vYwq46yBaqO5ZbP9TjUsATUNxrKeBdw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/types@4.1.0':
|
||||
resolution: {integrity: sha512-enhjdwp4D7CXmwLtD6zbcDMbo6/T6WtuuKCY49Xxc6OMOmUWlBEBDREsxxgV2LIdeQPW756+f97GzcgAwp3iLw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1738,6 +1787,10 @@ packages:
|
|||
resolution: {integrity: sha512-gPXcIEUtw7VlK8f/QcruNXm7q+T5hhvGu9tl63LsJPZ27exB6dtNwvh2HIi0v7JcXJ5emBxB+CJxwaLEdJfA+g==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/url-parser@4.0.2':
|
||||
resolution: {integrity: sha512-Bm8n3j2ScqnT+kJaClSVCMeiSenK6jVAzZCNewsYWuZtnBehEz4r2qP0riZySZVfzB+03XZHJeqfmJDkeeSLiQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/util-base64@4.0.0':
|
||||
resolution: {integrity: sha512-CvHfCmO2mchox9kjrtzoHkWHxjHZzaFojLc8quxXY7WAAMAg43nuxwv95tATVgQFNDwd4M9S1qFzj40Ul41Kmg==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1782,6 +1835,10 @@ packages:
|
|||
resolution: {integrity: sha512-HiLAvlcqhbzhuiOa0Lyct5IIlyIz0PQO5dnMlmQ/ubYM46dPInB+3yQGkfxsk6Q24Y0n3/JmcA1v5iEhmOF5mA==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/util-middleware@4.0.2':
|
||||
resolution: {integrity: sha512-6GDamTGLuBQVAEuQ4yDQ+ti/YINf/MEmIegrEeg7DdB/sld8BX1lqt9RRuIcABOhAGTA50bRbPzErez7SlDtDQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/util-retry@4.0.1':
|
||||
resolution: {integrity: sha512-WmRHqNVwn3kI3rKk1LsKcVgPBG6iLTBGC1iYOV3GQegwJ3E8yjzHytPt26VNzOWr1qu0xE03nK0Ug8S7T7oufw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -1790,6 +1847,10 @@ packages:
|
|||
resolution: {integrity: sha512-+Xvh8nhy0Wjv1y71rBVyV3eJU3356XsFQNI8dEZVNrQju7Eib8G31GWtO+zMa9kTCGd41Mflu+ZKfmQL/o2XzQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/util-stream@4.2.0':
|
||||
resolution: {integrity: sha512-Vj1TtwWnuWqdgQI6YTUF5hQ/0jmFiOYsc51CSMgj7QfyO+RF4EnT2HNjoviNlOOmgzgvf3f5yno+EiC4vrnaWQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/util-uri-escape@4.0.0':
|
||||
resolution: {integrity: sha512-77yfbCbQMtgtTylO9itEAdpPXSog3ZxMe09AEhm0dU0NLTalV70ghDZFR+Nfi1C60jnJoh/Re4090/DuZh2Omg==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
|
@ -2144,6 +2205,9 @@ packages:
|
|||
balanced-match@1.0.2:
|
||||
resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==}
|
||||
|
||||
base64-js@1.5.1:
|
||||
resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
|
||||
|
||||
binary-extensions@2.3.0:
|
||||
resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==}
|
||||
engines: {node: '>=8'}
|
||||
|
|
@ -2173,6 +2237,9 @@ packages:
|
|||
buffer-from@1.1.2:
|
||||
resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}
|
||||
|
||||
buffer@5.6.0:
|
||||
resolution: {integrity: sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==}
|
||||
|
||||
bundle-require@5.1.0:
|
||||
resolution: {integrity: sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==}
|
||||
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
|
||||
|
|
@ -2786,6 +2853,10 @@ packages:
|
|||
eventemitter3@4.0.7:
|
||||
resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==}
|
||||
|
||||
events@3.3.0:
|
||||
resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==}
|
||||
engines: {node: '>=0.8.x'}
|
||||
|
||||
express@4.21.2:
|
||||
resolution: {integrity: sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==}
|
||||
engines: {node: '>= 0.10.0'}
|
||||
|
|
@ -3069,6 +3140,9 @@ packages:
|
|||
resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
ieee754@1.2.1:
|
||||
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
|
||||
|
||||
ignore@5.3.2:
|
||||
resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==}
|
||||
engines: {node: '>= 4'}
|
||||
|
|
@ -4075,6 +4149,10 @@ packages:
|
|||
readable-stream@2.3.8:
|
||||
resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}
|
||||
|
||||
readable-stream@3.6.2:
|
||||
resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
|
||||
engines: {node: '>= 6'}
|
||||
|
||||
readdirp@3.6.0:
|
||||
resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==}
|
||||
engines: {node: '>=8.10.0'}
|
||||
|
|
@ -4326,6 +4404,9 @@ packages:
|
|||
resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}
|
||||
engines: {node: '>= 0.8'}
|
||||
|
||||
stream-browserify@3.0.0:
|
||||
resolution: {integrity: sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==}
|
||||
|
||||
streamsearch@1.1.0:
|
||||
resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==}
|
||||
engines: {node: '>=10.0.0'}
|
||||
|
|
@ -5024,28 +5105,28 @@ snapshots:
|
|||
'@aws-sdk/util-user-agent-browser': 3.723.0
|
||||
'@aws-sdk/util-user-agent-node': 3.726.0
|
||||
'@smithy/config-resolver': 4.0.1
|
||||
'@smithy/core': 3.1.4
|
||||
'@smithy/core': 3.2.0
|
||||
'@smithy/fetch-http-handler': 5.0.1
|
||||
'@smithy/hash-node': 4.0.1
|
||||
'@smithy/invalid-dependency': 4.0.1
|
||||
'@smithy/middleware-content-length': 4.0.1
|
||||
'@smithy/middleware-endpoint': 4.0.5
|
||||
'@smithy/middleware-endpoint': 4.1.0
|
||||
'@smithy/middleware-retry': 4.0.6
|
||||
'@smithy/middleware-serde': 4.0.2
|
||||
'@smithy/middleware-stack': 4.0.1
|
||||
'@smithy/node-config-provider': 4.0.1
|
||||
'@smithy/middleware-serde': 4.0.3
|
||||
'@smithy/middleware-stack': 4.0.2
|
||||
'@smithy/node-config-provider': 4.0.2
|
||||
'@smithy/node-http-handler': 4.0.4
|
||||
'@smithy/protocol-http': 5.0.1
|
||||
'@smithy/smithy-client': 4.1.5
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/url-parser': 4.0.1
|
||||
'@smithy/protocol-http': 5.1.0
|
||||
'@smithy/smithy-client': 4.2.0
|
||||
'@smithy/types': 4.2.0
|
||||
'@smithy/url-parser': 4.0.2
|
||||
'@smithy/util-base64': 4.0.0
|
||||
'@smithy/util-body-length-browser': 4.0.0
|
||||
'@smithy/util-body-length-node': 4.0.0
|
||||
'@smithy/util-defaults-mode-browser': 4.0.6
|
||||
'@smithy/util-defaults-mode-node': 4.0.6
|
||||
'@smithy/util-endpoints': 3.0.1
|
||||
'@smithy/util-middleware': 4.0.1
|
||||
'@smithy/util-middleware': 4.0.2
|
||||
'@smithy/util-retry': 4.0.1
|
||||
'@smithy/util-utf8': 4.0.0
|
||||
tslib: 2.8.1
|
||||
|
|
@ -5116,7 +5197,7 @@ snapshots:
|
|||
'@aws-sdk/core': 3.723.0
|
||||
'@aws-sdk/types': 3.723.0
|
||||
'@smithy/property-provider': 4.0.1
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@aws-sdk/credential-provider-http@3.723.0':
|
||||
|
|
@ -5126,9 +5207,9 @@ snapshots:
|
|||
'@smithy/fetch-http-handler': 5.0.1
|
||||
'@smithy/node-http-handler': 4.0.4
|
||||
'@smithy/property-provider': 4.0.1
|
||||
'@smithy/protocol-http': 5.0.1
|
||||
'@smithy/smithy-client': 4.1.5
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/protocol-http': 5.1.0
|
||||
'@smithy/smithy-client': 4.2.0
|
||||
'@smithy/types': 4.2.0
|
||||
'@smithy/util-stream': 4.1.1
|
||||
tslib: 2.8.1
|
||||
|
||||
|
|
@ -5145,7 +5226,7 @@ snapshots:
|
|||
'@smithy/credential-provider-imds': 4.0.1
|
||||
'@smithy/property-provider': 4.0.1
|
||||
'@smithy/shared-ini-file-loader': 4.0.1
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
transitivePeerDependencies:
|
||||
- '@aws-sdk/client-sso-oidc'
|
||||
|
|
@ -5176,7 +5257,7 @@ snapshots:
|
|||
'@aws-sdk/types': 3.723.0
|
||||
'@smithy/property-provider': 4.0.1
|
||||
'@smithy/shared-ini-file-loader': 4.0.1
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@aws-sdk/credential-provider-sso@3.726.0(@aws-sdk/client-sso-oidc@3.726.0(@aws-sdk/client-sts@3.726.1))':
|
||||
|
|
@ -5187,7 +5268,7 @@ snapshots:
|
|||
'@aws-sdk/types': 3.723.0
|
||||
'@smithy/property-provider': 4.0.1
|
||||
'@smithy/shared-ini-file-loader': 4.0.1
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
transitivePeerDependencies:
|
||||
- '@aws-sdk/client-sso-oidc'
|
||||
|
|
@ -5199,7 +5280,18 @@ snapshots:
|
|||
'@aws-sdk/core': 3.723.0
|
||||
'@aws-sdk/types': 3.723.0
|
||||
'@smithy/property-provider': 4.0.1
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@aws-sdk/lib-storage@3.726.1(@aws-sdk/client-s3@3.726.1)':
|
||||
dependencies:
|
||||
'@aws-sdk/client-s3': 3.726.1
|
||||
'@smithy/abort-controller': 4.0.2
|
||||
'@smithy/middleware-endpoint': 4.1.0
|
||||
'@smithy/smithy-client': 4.2.0
|
||||
buffer: 5.6.0
|
||||
events: 3.3.0
|
||||
stream-browserify: 3.0.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@aws-sdk/middleware-bucket-endpoint@3.726.0':
|
||||
|
|
@ -5317,8 +5409,8 @@ snapshots:
|
|||
'@aws-sdk/client-sso-oidc': 3.726.0(@aws-sdk/client-sts@3.726.1)
|
||||
'@aws-sdk/types': 3.723.0
|
||||
'@smithy/property-provider': 4.0.1
|
||||
'@smithy/shared-ini-file-loader': 4.0.1
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/shared-ini-file-loader': 4.0.2
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@aws-sdk/types@3.723.0':
|
||||
|
|
@ -5328,7 +5420,7 @@ snapshots:
|
|||
|
||||
'@aws-sdk/types@3.734.0':
|
||||
dependencies:
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@aws-sdk/util-arn-parser@3.723.0':
|
||||
|
|
@ -6353,7 +6445,7 @@ snapshots:
|
|||
|
||||
'@smithy/abort-controller@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/abort-controller@4.0.2':
|
||||
|
|
@ -6389,18 +6481,29 @@ snapshots:
|
|||
'@smithy/util-utf8': 4.0.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/core@3.2.0':
|
||||
dependencies:
|
||||
'@smithy/middleware-serde': 4.0.3
|
||||
'@smithy/protocol-http': 5.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
'@smithy/util-body-length-browser': 4.0.0
|
||||
'@smithy/util-middleware': 4.0.2
|
||||
'@smithy/util-stream': 4.2.0
|
||||
'@smithy/util-utf8': 4.0.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/credential-provider-imds@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/node-config-provider': 4.0.1
|
||||
'@smithy/property-provider': 4.0.1
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
'@smithy/url-parser': 4.0.1
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/eventstream-codec@4.0.1':
|
||||
dependencies:
|
||||
'@aws-crypto/crc32': 5.2.0
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
'@smithy/util-hex-encoding': 4.0.0
|
||||
tslib: 2.8.1
|
||||
|
||||
|
|
@ -6424,7 +6527,7 @@ snapshots:
|
|||
'@smithy/eventstream-serde-universal@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/eventstream-codec': 4.0.1
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/fetch-http-handler@5.0.1':
|
||||
|
|
@ -6435,6 +6538,14 @@ snapshots:
|
|||
'@smithy/util-base64': 4.0.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/fetch-http-handler@5.0.2':
|
||||
dependencies:
|
||||
'@smithy/protocol-http': 5.1.0
|
||||
'@smithy/querystring-builder': 4.0.2
|
||||
'@smithy/types': 4.2.0
|
||||
'@smithy/util-base64': 4.0.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/hash-blob-browser@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/chunked-blob-reader': 5.0.0
|
||||
|
|
@ -6491,6 +6602,17 @@ snapshots:
|
|||
'@smithy/util-middleware': 4.0.1
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/middleware-endpoint@4.1.0':
|
||||
dependencies:
|
||||
'@smithy/core': 3.2.0
|
||||
'@smithy/middleware-serde': 4.0.3
|
||||
'@smithy/node-config-provider': 4.0.2
|
||||
'@smithy/shared-ini-file-loader': 4.0.2
|
||||
'@smithy/types': 4.2.0
|
||||
'@smithy/url-parser': 4.0.2
|
||||
'@smithy/util-middleware': 4.0.2
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/middleware-retry@4.0.6':
|
||||
dependencies:
|
||||
'@smithy/node-config-provider': 4.0.1
|
||||
|
|
@ -6508,11 +6630,21 @@ snapshots:
|
|||
'@smithy/types': 4.1.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/middleware-serde@4.0.3':
|
||||
dependencies:
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/middleware-stack@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/types': 4.1.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/middleware-stack@4.0.2':
|
||||
dependencies:
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/node-config-provider@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/property-provider': 4.0.1
|
||||
|
|
@ -6520,6 +6652,13 @@ snapshots:
|
|||
'@smithy/types': 4.1.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/node-config-provider@4.0.2':
|
||||
dependencies:
|
||||
'@smithy/property-provider': 4.0.2
|
||||
'@smithy/shared-ini-file-loader': 4.0.2
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/node-http-handler@4.0.4':
|
||||
dependencies:
|
||||
'@smithy/abort-controller': 4.0.2
|
||||
|
|
@ -6530,7 +6669,12 @@ snapshots:
|
|||
|
||||
'@smithy/property-provider@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/property-provider@4.0.2':
|
||||
dependencies:
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/protocol-http@5.0.1':
|
||||
|
|
@ -6545,7 +6689,7 @@ snapshots:
|
|||
|
||||
'@smithy/querystring-builder@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
'@smithy/util-uri-escape': 4.0.0
|
||||
tslib: 2.8.1
|
||||
|
||||
|
|
@ -6557,23 +6701,33 @@ snapshots:
|
|||
|
||||
'@smithy/querystring-parser@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/querystring-parser@4.0.2':
|
||||
dependencies:
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/service-error-classification@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
|
||||
'@smithy/shared-ini-file-loader@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/shared-ini-file-loader@4.0.2':
|
||||
dependencies:
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/signature-v4@5.0.1':
|
||||
dependencies:
|
||||
'@smithy/is-array-buffer': 4.0.0
|
||||
'@smithy/protocol-http': 5.0.1
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/protocol-http': 5.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
'@smithy/util-hex-encoding': 4.0.0
|
||||
'@smithy/util-middleware': 4.0.1
|
||||
'@smithy/util-uri-escape': 4.0.0
|
||||
|
|
@ -6590,6 +6744,16 @@ snapshots:
|
|||
'@smithy/util-stream': 4.1.1
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/smithy-client@4.2.0':
|
||||
dependencies:
|
||||
'@smithy/core': 3.2.0
|
||||
'@smithy/middleware-endpoint': 4.1.0
|
||||
'@smithy/middleware-stack': 4.0.2
|
||||
'@smithy/protocol-http': 5.1.0
|
||||
'@smithy/types': 4.2.0
|
||||
'@smithy/util-stream': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/types@4.1.0':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
|
@ -6604,6 +6768,12 @@ snapshots:
|
|||
'@smithy/types': 4.1.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/url-parser@4.0.2':
|
||||
dependencies:
|
||||
'@smithy/querystring-parser': 4.0.2
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/util-base64@4.0.0':
|
||||
dependencies:
|
||||
'@smithy/util-buffer-from': 4.0.0
|
||||
|
|
@ -6665,6 +6835,11 @@ snapshots:
|
|||
'@smithy/types': 4.1.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/util-middleware@4.0.2':
|
||||
dependencies:
|
||||
'@smithy/types': 4.2.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/util-retry@4.0.1':
|
||||
dependencies:
|
||||
'@smithy/service-error-classification': 4.0.1
|
||||
|
|
@ -6682,6 +6857,17 @@ snapshots:
|
|||
'@smithy/util-utf8': 4.0.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/util-stream@4.2.0':
|
||||
dependencies:
|
||||
'@smithy/fetch-http-handler': 5.0.2
|
||||
'@smithy/node-http-handler': 4.0.4
|
||||
'@smithy/types': 4.2.0
|
||||
'@smithy/util-base64': 4.0.0
|
||||
'@smithy/util-buffer-from': 4.0.0
|
||||
'@smithy/util-hex-encoding': 4.0.0
|
||||
'@smithy/util-utf8': 4.0.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/util-uri-escape@4.0.0':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
|
@ -7090,6 +7276,8 @@ snapshots:
|
|||
|
||||
balanced-match@1.0.2: {}
|
||||
|
||||
base64-js@1.5.1: {}
|
||||
|
||||
binary-extensions@2.3.0: {}
|
||||
|
||||
body-parser@1.20.3:
|
||||
|
|
@ -7133,6 +7321,11 @@ snapshots:
|
|||
|
||||
buffer-from@1.1.2: {}
|
||||
|
||||
buffer@5.6.0:
|
||||
dependencies:
|
||||
base64-js: 1.5.1
|
||||
ieee754: 1.2.1
|
||||
|
||||
bundle-require@5.1.0(esbuild@0.24.2):
|
||||
dependencies:
|
||||
esbuild: 0.24.2
|
||||
|
|
@ -7859,6 +8052,8 @@ snapshots:
|
|||
|
||||
eventemitter3@4.0.7: {}
|
||||
|
||||
events@3.3.0: {}
|
||||
|
||||
express@4.21.2:
|
||||
dependencies:
|
||||
accepts: 1.3.8
|
||||
|
|
@ -8247,6 +8442,8 @@ snapshots:
|
|||
dependencies:
|
||||
safer-buffer: 2.1.2
|
||||
|
||||
ieee754@1.2.1: {}
|
||||
|
||||
ignore@5.3.2: {}
|
||||
|
||||
immutable@5.1.1: {}
|
||||
|
|
@ -9454,6 +9651,12 @@ snapshots:
|
|||
string_decoder: 1.1.1
|
||||
util-deprecate: 1.0.2
|
||||
|
||||
readable-stream@3.6.2:
|
||||
dependencies:
|
||||
inherits: 2.0.4
|
||||
string_decoder: 1.1.1
|
||||
util-deprecate: 1.0.2
|
||||
|
||||
readdirp@3.6.0:
|
||||
dependencies:
|
||||
picomatch: 2.3.1
|
||||
|
|
@ -9792,6 +9995,11 @@ snapshots:
|
|||
|
||||
statuses@2.0.1: {}
|
||||
|
||||
stream-browserify@3.0.0:
|
||||
dependencies:
|
||||
inherits: 2.0.4
|
||||
readable-stream: 3.6.2
|
||||
|
||||
streamsearch@1.1.0: {}
|
||||
|
||||
string-width@4.2.3:
|
||||
|
|
|
|||
|
|
@ -1,22 +1,20 @@
|
|||
import { bytes } from '@/lib/bytes';
|
||||
import { reloadSettings } from '@/lib/config';
|
||||
import { prisma } from '@/lib/db';
|
||||
import { fileSelect } from '@/lib/db/models/file';
|
||||
import { userSelect } from '@/lib/db/models/user';
|
||||
import { onUpload } from '@/lib/webhooks';
|
||||
import { log } from '@/lib/logger';
|
||||
import { UploadOptions } from '@/lib/uploader/parseHeaders';
|
||||
import { open, readFile, readdir, rm } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { isMainThread, workerData } from 'worker_threads';
|
||||
import { createReadStream } from 'fs';
|
||||
import {
|
||||
CompleteMultipartUploadCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
UploadPartCommand,
|
||||
} from '@aws-sdk/client-s3';
|
||||
import { getDatasource } from '@/lib/datasource';
|
||||
import { S3Datasource } from '@/lib/datasource/S3';
|
||||
import { prisma } from '@/lib/db';
|
||||
import { fileSelect } from '@/lib/db/models/file';
|
||||
import { IncompleteFile } from '@/lib/db/models/incompleteFile';
|
||||
import { userSelect } from '@/lib/db/models/user';
|
||||
import { log } from '@/lib/logger';
|
||||
import { randomCharacters } from '@/lib/random';
|
||||
import { UploadOptions } from '@/lib/uploader/parseHeaders';
|
||||
import { onUpload } from '@/lib/webhooks';
|
||||
import { Upload } from '@aws-sdk/lib-storage';
|
||||
import { createReadStream, createWriteStream } from 'fs';
|
||||
import { open, readdir, rm } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { isMainThread, workerData } from 'worker_threads';
|
||||
|
||||
export type PartialWorkerData = {
|
||||
user: {
|
||||
|
|
@ -50,31 +48,33 @@ if (!options.partial.lastchunk) {
|
|||
process.exit(1);
|
||||
}
|
||||
|
||||
worker();
|
||||
main();
|
||||
|
||||
async function worker() {
|
||||
async function main() {
|
||||
await reloadSettings();
|
||||
|
||||
const config = global.__config__;
|
||||
getDatasource(config);
|
||||
|
||||
const datasource = global.__datasource__;
|
||||
|
||||
if (!config.chunks.enabled) {
|
||||
logger.error('chunks are not enabled');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const datasource = global.__datasource__;
|
||||
|
||||
logger.debug('started partial upload worker');
|
||||
|
||||
const partials = await readdir(config.core.tempDirectory).then((files) =>
|
||||
files.filter((file) => file.startsWith(`zipline_partial_${options.partial!.identifier}`)),
|
||||
);
|
||||
|
||||
const readChunks = partials.map((file) => {
|
||||
const [, , , start, end] = file.split('_');
|
||||
return { file, start: Number(start), end: Number(end) };
|
||||
});
|
||||
const readChunks = partials
|
||||
.map((file) => {
|
||||
const [, , , start, end] = file.split('_');
|
||||
return { file, start: Number(start), end: Number(end) };
|
||||
})
|
||||
.sort((a, b) => a.start - b.start);
|
||||
|
||||
const incompleteFile = await prisma.incompleteFile.create({
|
||||
data: {
|
||||
|
|
@ -88,17 +88,32 @@ async function worker() {
|
|||
},
|
||||
});
|
||||
|
||||
if (config.datasource.type === 'local') {
|
||||
const fd = await open(join(config.datasource.local!.directory, file.filename), 'w');
|
||||
const finalPath =
|
||||
config.datasource.type === 'local'
|
||||
? join(config.datasource.local!.directory, file.filename)
|
||||
: join(config.core.tempDirectory, randomCharacters(16));
|
||||
|
||||
for (let i = 0; i !== readChunks.length; ++i) {
|
||||
const chunk = readChunks[i];
|
||||
const fd = await open(finalPath, 'w');
|
||||
await fd.close();
|
||||
|
||||
const buffer = await readFile(join(config.core.tempDirectory, chunk.file));
|
||||
for (let i = 0; i !== readChunks.length; ++i) {
|
||||
const chunk = readChunks[i];
|
||||
|
||||
const { bytesWritten } = await fd.write(buffer, 0, buffer.length, chunk.start);
|
||||
const chunkPath = join(config.core.tempDirectory, chunk.file);
|
||||
|
||||
await rm(join(config.core.tempDirectory, chunk.file));
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const readStream = createReadStream(chunkPath);
|
||||
const writeStream = createWriteStream(finalPath, { start: chunk.start, flags: 'r+' });
|
||||
|
||||
readStream.pipe(writeStream);
|
||||
|
||||
writeStream.on('finish', resolve);
|
||||
|
||||
writeStream.on('error', reject);
|
||||
});
|
||||
|
||||
await rm(chunkPath);
|
||||
await prisma.incompleteFile.update({
|
||||
where: {
|
||||
id: incompleteFile.id,
|
||||
|
|
@ -112,86 +127,47 @@ async function worker() {
|
|||
});
|
||||
|
||||
logger.debug(`wrote chunk ${i + 1}/${readChunks.length}`, {
|
||||
bytesWritten,
|
||||
start: chunk.start,
|
||||
end: chunk.end,
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
logger.error('error while combining chunks');
|
||||
console.error(e);
|
||||
await failPartial(incompleteFile);
|
||||
|
||||
await fd.close();
|
||||
} else if (config.datasource.type === 's3') {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (config.datasource.type === 's3') {
|
||||
logger.debug('starting multipart upload process for s3');
|
||||
|
||||
const bodyStream = createReadStream(finalPath);
|
||||
const s3datasource = datasource as S3Datasource;
|
||||
const { UploadId } = await s3datasource.client.send(
|
||||
new CreateMultipartUploadCommand({ Bucket: s3datasource.options.bucket, Key: file.filename }),
|
||||
);
|
||||
|
||||
const partResults = [];
|
||||
|
||||
for (let i = 0; i !== readChunks.length; ++i) {
|
||||
const chunk = readChunks[i];
|
||||
|
||||
const stream = createReadStream(join(config.core.tempDirectory, chunk.file));
|
||||
|
||||
try {
|
||||
const res = await s3datasource.client.send(
|
||||
new UploadPartCommand({
|
||||
Bucket: s3datasource.options.bucket,
|
||||
Key: file.filename,
|
||||
UploadId,
|
||||
PartNumber: i + 1,
|
||||
Body: stream,
|
||||
ContentLength: chunk.end - chunk.start,
|
||||
}),
|
||||
);
|
||||
|
||||
logger.debug(`uploaded chunk to s3 ${i + 1}/${readChunks.length}`, {
|
||||
ETag: res.ETag,
|
||||
start: chunk.start,
|
||||
end: chunk.end,
|
||||
});
|
||||
|
||||
partResults.push({
|
||||
ETag: res.ETag,
|
||||
PartNumber: i + 1,
|
||||
});
|
||||
} catch (e) {
|
||||
logger.error('error while uploading chunk');
|
||||
console.error(e);
|
||||
return;
|
||||
} finally {
|
||||
await rm(join(config.core.tempDirectory, chunk.file));
|
||||
|
||||
await prisma.incompleteFile.update({
|
||||
where: {
|
||||
id: incompleteFile.id,
|
||||
},
|
||||
data: {
|
||||
chunksComplete: {
|
||||
increment: 1,
|
||||
},
|
||||
status: 'PROCESSING',
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await s3datasource.client.send(
|
||||
new CompleteMultipartUploadCommand({
|
||||
const upload = new Upload({
|
||||
client: s3datasource.client,
|
||||
params: {
|
||||
Bucket: s3datasource.options.bucket,
|
||||
Key: file.filename,
|
||||
UploadId,
|
||||
MultipartUpload: {
|
||||
Parts: partResults,
|
||||
},
|
||||
}),
|
||||
);
|
||||
Body: bodyStream,
|
||||
},
|
||||
partSize: bytes(config.chunks.size),
|
||||
leavePartsOnError: false,
|
||||
});
|
||||
|
||||
logger.debug('completed multipart upload for s3');
|
||||
upload.on('httpUploadProgress', (progress) => logger.debug('s3 MultipartUpload', { ...progress }));
|
||||
|
||||
await upload.done();
|
||||
|
||||
await rm(finalPath);
|
||||
} catch (e) {
|
||||
logger.error('error while completing multipart upload');
|
||||
logger.error('error while uploading multipart file');
|
||||
console.error(e);
|
||||
return;
|
||||
await failPartial(incompleteFile);
|
||||
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -239,3 +215,15 @@ async function runComplete(id: string) {
|
|||
},
|
||||
});
|
||||
}
|
||||
|
||||
function failPartial(incompleteFile: IncompleteFile) {
|
||||
logger.error('failing incomplete file', { id: incompleteFile.id });
|
||||
return prisma.incompleteFile.update({
|
||||
where: {
|
||||
id: incompleteFile.id,
|
||||
},
|
||||
data: {
|
||||
status: 'FAILED',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue