Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# no need to call __load_local_json() again as __auth() will load the json & acess token.
result = self.__auth()
if result != const.ENoError:
perr("Program authorization FAILED.\n"
"You need to authorize this program before using any PCS functions.\n"
"Quitting...\n")
self.quit(result)
for proxy in ['HTTP_PROXY', 'HTTPS_PROXY']:
if proxy in os.environ:
pr("{} used: {}".format(proxy, os.environ[proxy]))
# update check
check_update = False
nowsec = int(time.time())
if const.SettingKey_LastUpdateCheckTime not in self.__setting:
check_update = True
else:
lastcheck = self.__setting[const.SettingKey_LastUpdateCheckTime]
if nowsec - lastcheck > 7 * 24 * 60 * 60: # check every 7 days
check_update = True
if check_update:
r = requests.get('https://raw.githubusercontent.com/houtianze/bypy/master/update/update.json')
if r.status_code == 200:
try:
j = r.json()
min_ver_key = 'minimumRequiredVersion'
if min_ver_key in j:
minver = j[min_ver_key]
if comp_semver(const.__version__, minver) < 0:
perr("Your current version ({}) is too low, "
"not properly installed (missing C extensions). "
"You need to install a C compiler and Python headers before "
"installing the Python 'multiprocess' library. "
"(All these hassles could have been saved if Python's builtin "
"'multiprocessing' works properly, sigh.)\n"
"Fix for debian derivatives:\n"
"- Install gcc: # apt-get install gcc\n"
"- Install python-dev: # apt-get install python-dev\n"
"- Reinstall Python 'multiprocess' library:\n"
" # pip uninstall -y multiprocess\n"
" # pip install -v multiprocess\n"
"- If there's no errors/warnings in the above actions, "
"then this error should be gone when you run 'bypy' with '{}' again.\n"
).format(formatex(pe), const.MultiprocessOption)
perr(errmsg)
self.quit(const.EFatal)
parser.add_argument("--select-fastest-mirror",
dest="selectmirror", action="store_true",
help="Let the program run some tests and select the fastest PCS mirror it detectes. [default: %(default)s]")
parser.add_argument("--rapid-upload-only",
dest="rapiduploadonly", action="store_true",
help="only upload large files that can be rapidly uploaded")
parser.add_argument("--resume-download-revert-back",
dest="resumedl_revertcount", default=const.DefaultResumeDlRevertCount,
type=int, metavar='RCOUNT',
help="Revert back at least %(metavar)s download chunk(s) and align to chunk boundary when resuming the download. A negative value means NO reverts. [default: %(default)s]")
parser.add_argument("--move",
dest="deletesource", action="store_true",
help="Delete source files/directories after download/upload/syncdown/syncup is successful (This will force verification of the files). [default: %(default)s]")
if Pool:
parser.add_argument(const.MultiprocessOption,
dest="processes", default=const.DefaultProcessCount, type=int,
help="Number of parallel processes. (Only applies to dir sync/dl/ul). [default: %(default)s]")
# support aria2c
parser.add_argument("--downloader",
dest="downloader", default="",
help="downloader to use (use python if not specified). valid values: {} [default: %(default)s]".format(const.Downloaders))
parser.add_argument("--downloader-arguments",
dest="downloader_args", default="",
help=("arguments for the downloader:\n"
"normally, the string is the arguments to be passed to the downloader. "
"however, when it begins with '{0}', it will be treated as the name of file, "
"whose contents will be used as the downloader arguments "
"(example: when specifying '{0}args.txt', file contents of 'args.txt' will be "
"used as the downloader arguments, not the string '@args.txt' itself).\n"
"you can also use environment variable '{1}' to specify the downloader arguments "
"(the environment variable has lower priority compared to this argument).\n"
def __walk_remote_dir(self, remotepath, remoterootpath, args = None, skip_remote_only_dirs = False):
dirjs = []
filejs = []
listStart = 0
# https://github.com/houtianze/bypy/issues/285
while True:
pars = {
'method' : 'list',
'path' : remotepath,
'by' : 'name',
'order' : 'asc',
'limit' : '{}-{}'.format(listStart, listStart + const.MaxListEntries)}
# Python parameters are by-reference and mutable, so they are 'out' by default
result = self.__get(pcsurl + 'file', pars, self.__walk_proceed_remote_dir_act, (dirjs, filejs))
if len(dirjs) + len(filejs) - listStart < const.MaxListEntries:
break
listStart += const.MaxListEntries
yield [result, remotepath, dirjs, filejs, args]
if result == const.ENoError:
self.pd("Remote dirs: {}".format(dirjs))
self.pd("Remote files: {}".format(filejs))
for dirj in dirjs:
crpath = dirj['path'] # crpath - current remote path
if skip_remote_only_dirs and remoterootpath != None and \
self.__local_dir_contents.get(posixpath.relpath(crpath, remoterootpath)) == None:
self.pd("Skipping remote-only sub-directory '{}'.".format(crpath))
continue
# http://stackoverflow.com/a/8991864/404271
# for Python 3.3+, "yield from" reads better
for y in self.__walk_remote_dir(crpath, remoterootpath, args, skip_remote_only_dirs):
return const.EParameter
if fast:
self.__get_hashes_for_rapidupload(lpath, setlocalfile = True)
pr(self.__get_accept_cmd(rpath))
return const.ENoError
ulrpath = const.RemoteTempDir + '/' + posixpath.basename(lpath)
result = self.__upload_file(lpath, ulrpath)
if result != const.ENoError:
perr("Unable to share as uploading failed")
return result
if not self.__rapiduploaded:
i = 0
while i < const.ShareRapidUploadRetries:
i += 1
result = self.__rapidupload_file(lpath, ulrpath, setlocalfile = True)
if result == const.ENoError: # or result == IEMD5NotFound: # retrying if MD5 not found _may_ make the file available?
break
else:
self.pd("Retrying #{} for sharing '{}'".format(i, lpath))
time.sleep(1)
if result == const.ENoError:
pr(self.__get_accept_cmd(rpath))
return const.ENoError
elif result == const.IEMD5NotFound:
pr("# Sharing (RapidUpload) not possible for '{}', error: {}".format(lpath, result))
return result
else:
pr("# Error sharing '{}', error: {}".format(lpath, result))
help="regular expression of files to include. if not specified (default), everything is included. for download, the regex applies to the remote files; for upload, the regex applies to the local files. to exclude files, think about your regex, some tips here: https://stackoverflow.com/questions/406230/regular-expression-to-match-string-not-containing-a-word [default: %(default)s]")
parser.add_argument("--on-dup",
dest="ondup", default='overwrite',
help="what to do when the same file / folder exists in the destination: 'overwrite', 'skip', 'prompt' [default: %(default)s]")
parser.add_argument("--no-symlink",
dest="followlink", action="store_false",
help="DON'T follow symbol links when uploading / syncing up")
parser.add_argument(const.DisableSslCheckOption,
dest="checkssl", action="store_false",
help="DON'T verify host SSL cerificate")
parser.add_argument(const.CaCertsOption,
dest="cacerts", default=None,
help="Specify the path for CA Bundle [default: %(default)s]")
parser.add_argument("--mirror",
dest="mirror", default='',
help="Specify the PCS mirror (e.g. bj.baidupcs.com. Open 'https://pcs.baidu.com/rest/2.0/pcs/manage?method=listhost' to get the list) to use. [default: " + const.PcsDomain + "]")
parser.add_argument("--select-fastest-mirror",
dest="selectmirror", action="store_true",
help="Let the program run some tests and select the fastest PCS mirror it detectes. [default: %(default)s]")
parser.add_argument("--rapid-upload-only",
dest="rapiduploadonly", action="store_true",
help="only upload large files that can be rapidly uploaded")
parser.add_argument("--resume-download-revert-back",
dest="resumedl_revertcount", default=const.DefaultResumeDlRevertCount,
type=int, metavar='RCOUNT',
help="Revert back at least %(metavar)s download chunk(s) and align to chunk boundary when resuming the download. A negative value means NO reverts. [default: %(default)s]")
parser.add_argument("--move",
dest="deletesource", action="store_true",
help="Delete source files/directories after download/upload/syncdown/syncup is successful (This will force verification of the files). [default: %(default)s]")
if Pool:
parser.add_argument(const.MultiprocessOption,
dest="processes", default=const.DefaultProcessCount, type=int,
if Pool != None:
if sys.version_info[0] == 2:
# enable with statement
from contextlib import contextmanager
@contextmanager
def UPool(*args, **kwargs):
yield Pool(*args, **kwargs)
elif sys.version_info[0] == 3:
UPool = Pool
# global instance for non-member function to access
gbypyinst = None
pcsurl = const.PcsUrl
cpcsurl = const.CPcsUrl
dpcsurl = const.DPcsUrl
# This crap is here to avoid circular imports
# What a fantastic packing architecture Python has!
def makemppr(pr):
def mppr(msg, *args, **kwargs):
return pr(mp.current_process().name + ': ' + msg, *args, **kwargs)
return mppr
def makemppprgr(pprgr):
def mppprgr(finish, total, start_time = None, existing = 0,
prefix = '', suffix = '', seg = 20):
prefix = mp.current_process().name + ': ' + str(prefix)
pprgr(finish, total, start_time, existing, prefix, suffix, seg)
return mppprgr
def set_mp_print():
def removedir(path, verbose = False):
result = const.ENoError
try:
if verbose:
pr("Removing local directory '{}'".format(path))
if path:
shutil.rmtree(path)
except Exception as ex:
perr("Fail to remove local directory '{}'.\n{}".format(
path, formatex(ex)))
result = const.EFailToDeleteDir
return result