58 lines
1.9 KiB
Python
58 lines
1.9 KiB
Python
|
# Copyright 2017 The Chromium Authors. All rights reserved.
|
||
|
|
# Use of this source code is governed by a BSD-style license that can be
|
||
|
|
# found in the LICENSE file.
|
||
|
|
|
||
|
|
|
||
|
|
from recipe_engine import recipe_api
|
||
|
|
|
||
|
|
UPLOAD_ATTEMPTS = 5
|
||
|
|
|
||
|
|
class GSUtilApi(recipe_api.RecipeApi):
|
||
|
|
def __call__(self, step_name, *args):
|
||
|
|
"""Run gsutil with the given args.
|
||
|
|
|
||
|
|
This assumes there exists an executable called gsutil on the PATH.
|
||
|
|
This probably only works for Linux/Mac, but those are the only
|
||
|
|
hosts that we try to upload to GCS from anyway.
|
||
|
|
"""
|
||
|
|
return self.m.step(step_name, cmd=['gsutil'] + list(args))
|
||
|
|
|
||
|
|
def cp(self, name, src, dst, extra_gsutil_args=None, extra_args=None,
|
||
|
|
multithread=False):
|
||
|
|
"""Attempt to upload or download files to/from Google Cloud Storage (GCS).
|
||
|
|
|
||
|
|
Args:
|
||
|
|
name: string. Will be used to fill out the step name.
|
||
|
|
src: string. Absolute path for a local file or gcs file (e.g. gs://...)
|
||
|
|
dst: string. Same as src.
|
||
|
|
extra_gsutil_args: optional list of args to be passed to gsutil before the
|
||
|
|
cp command.
|
||
|
|
extra_args: optional list of args to be passed to gsutil cp. e.g. [-Z]
|
||
|
|
asks all files be compressed with gzip after upload and before download.
|
||
|
|
multi_thread: if the -m argument should be used to copy multiple items
|
||
|
|
at once (e.g. gsutil -m cp foo* gs://bar/dir)
|
||
|
|
|
||
|
|
If the operation fails, it will be retried multiple times.
|
||
|
|
"""
|
||
|
|
cmd = []
|
||
|
|
if extra_gsutil_args:
|
||
|
|
cmd.extend(extra_gsutil_args)
|
||
|
|
if multithread:
|
||
|
|
cmd.append('-m')
|
||
|
|
cmd.append('cp')
|
||
|
|
if extra_args:
|
||
|
|
cmd.extend(extra_args)
|
||
|
|
cmd.extend([src, dst])
|
||
|
|
|
||
|
|
name = 'upload %s' % name
|
||
|
|
for i in range(UPLOAD_ATTEMPTS):
|
||
|
|
step_name = name
|
||
|
|
if i > 0:
|
||
|
|
step_name += ' (attempt %d)' % (i+1)
|
||
|
|
try:
|
||
|
|
self(step_name, *cmd)
|
||
|
|
break
|
||
|
|
except self.m.step.StepFailure:
|
||
|
|
if i == UPLOAD_ATTEMPTS - 1:
|
||
|
|
raise
|