NASA Ocean Color

ocssw V2022
 #!/usr/bin/env python3
  
 import argparse
 import hashlib
 import json
 import os
 import os.path
 import re
 import sys
 import urllib.request
 import subprocess
 import shutil
 import logging
 from contextlib import closing
 from datetime import datetime, timedelta, date
 import requests
 from requests.adapters import HTTPAdapter
 import tempfile
  
 DEFAULT_BASE_URL = "https://oceandata.sci.gsfc.nasa.gov/manifest/tags"
 MANIFEST_BASENAME = "manifest.json"
  
  
 #  ------------------ DANGER -------------------
 #
 # The next 5 functions:
 #    getSession
 #    isRequestAuthFailure
 #    httpdl
 #    uncompressFile
 #    get_file_time
 #
 # exist in two places:
 #    OCSSWROOT/src/manifest/manifest.py
 #    OCSSWROOT/src/scripts/seadasutils/ProcUtils.py
 #
 # Make sure changes get into both files.
 #
  
 DEFAULT_CHUNK_SIZE = 131072
  
 # requests session object used to keep connections around
 obpgSession = None
  
 def getSession(verbose=0, ntries=5):
     global obpgSession
  
     if not obpgSession:
         # turn on debug statements for requests
         if verbose > 1:
             logging.basicConfig(level=logging.DEBUG)
  
         obpgSession = requests.Session()
         obpgSession.mount('https://', HTTPAdapter(max_retries=ntries))
  
         if verbose:
             print("OBPG session started")
     else:
         if verbose > 1:
             print("reusing existing OBPG session")
  
     return obpgSession
  
 #  ------------------ DANGER -------------------
 # See comment above
 def isRequestAuthFailure(req) :
     ctype = req.headers.get('Content-Type')
     if ctype and ctype.startswith('text/html'):
         if "<title>Earthdata Login</title>" in req.text:
             return True
     return False
  
 #  ------------------ DANGER -------------------
 # See comment above
 def httpdl(server, request, localpath='.', outputfilename=None, ntries=5,
            uncompress=False, timeout=30., verbose=0, force_download=False,
            chunk_size=DEFAULT_CHUNK_SIZE):
  
     status = 0
     urlStr = 'https://' + server + request
  
     global obpgSession
  
     getSession(verbose=verbose, ntries=ntries)
  
     modified_since = None
     headers = {}
  
     if not force_download:
         if outputfilename:
             ofile = os.path.join(localpath, outputfilename)
             modified_since = get_file_time(ofile)
         else:
             ofile = os.path.join(localpath, os.path.basename(request.rstrip()))
             modified_since = get_file_time(ofile)
  
         if modified_since:
             headers = {"If-Modified-Since":modified_since.strftime("%a, %d %b %Y %H:%M:%S GMT")}
  
     with closing(obpgSession.get(urlStr, stream=True, timeout=timeout, headers=headers)) as req:
  
         if req.status_code != 200:
             status = req.status_code
         elif isRequestAuthFailure(req):
             status = 401
         else:
             if not os.path.exists(localpath):
                 os.umask(0o02)
                 os.makedirs(localpath, mode=0o2775)
  
             if not outputfilename:
                 cd = req.headers.get('Content-Disposition')
                 if cd:
                     outputfilename = re.findall("filename=(.+)", cd)[0]
                 else:
                     outputfilename = urlStr.split('/')[-1]
  
             ofile = os.path.join(localpath, outputfilename)
  
             # This is here just in case we didn't get a 304 when we should have...
             download = True
             if 'last-modified' in req.headers:
                 remote_lmt = req.headers['last-modified']
                 remote_ftime = datetime.strptime(remote_lmt, "%a, %d %b %Y %H:%M:%S GMT").replace(tzinfo=None)
                 if modified_since and not force_download:
                     if (remote_ftime - modified_since).total_seconds() < 0:
                         download = False
                         if verbose:
                             print("Skipping download of %s" % outputfilename)
  
             if download:
                 with open(ofile, 'wb') as fd:
                     for chunk in req.iter_content(chunk_size=chunk_size):
                         if chunk: # filter out keep-alive new chunks
                             fd.write(chunk)
  
                 if uncompress and re.search(".(Z|gz|bz2)$", ofile):
                     compressStatus = uncompressFile(ofile)
                     if compressStatus:
                         status = compressStatus
                 else:
                     status = 0
  
     return status
  
  
 #  ------------------ DANGER -------------------
 # See comment above
 def uncompressFile(compressed_file):
     """
     uncompress file
     compression methods:
         bzip2
         gzip
         UNIX compress
     """
  
     compProg = {"gz": "gunzip -f ", "Z": "gunzip -f ", "bz2": "bunzip2 -f "}
     exten = os.path.basename(compressed_file).split('.')[-1]
     unzip = compProg[exten]
     p = subprocess.Popen(unzip + compressed_file, shell=True)
     status = os.waitpid(p.pid, 0)[1]
     if status:
         print("Warning! Unable to decompress %s" % compressed_file)
         return status
     else:
         return 0
  
 #  ------------------ DANGER -------------------
 # See comment above
 def get_file_time(localFile):
     ftime = None
     if not os.path.isfile(localFile):
         localFile = re.sub(r".(Z|gz|bz2)$", '', localFile)
  
     if os.path.isfile(localFile):
         ftime = datetime.fromtimestamp(os.path.getmtime(localFile))
  
     return ftime
  
 def run():
     parser = argparse.ArgumentParser()
     parser.set_defaults(func=download)
     subparsers = parser.add_subparsers()
  
     _add_subparser_reprint(subparsers)
     _add_subparser_update_file(subparsers)
     _add_subparser_add_tag(subparsers)
     _add_subparser_get_value(subparsers)
     _add_subparser_get_first_tag(subparsers)
     _add_subparser_list(subparsers)
     _add_subparser_clean(subparsers)
     _add_subparser_download(subparsers)
     _add_subparser_generate(subparsers)
     _add_subparser_list_tags(subparsers)
  
     options, args = parser.parse_known_args()
     return options.func(options, args)
  
 def _add_subparser_reprint(subparsers):
     parser_reprint = subparsers.add_parser('reprint')
     parser_reprint.add_argument("manifest", help="manifest to reprint")
     parser_reprint.set_defaults(func=reprint)
     if os.path.isfile(MANIFEST_BASENAME):
         parser_reprint.set_defaults(manifest=MANIFEST_BASENAME)
  
 def _add_subparser_update_file(subparsers):
     parser_update_file = subparsers.add_parser('update-file')
     parser_update_file.add_argument("manifest", help="manifest to update")
     parser_update_file.add_argument("path", help="file to update")
     parser_update_file.set_defaults(func=update_file)
     if os.path.isfile(MANIFEST_BASENAME):
         parser_update_file.set_defaults(manifest=MANIFEST_BASENAME)
  
 def _add_subparser_add_tag(subparsers):
     parser_add_tag = subparsers.add_parser('add-tag')
     parser_add_tag.add_argument("-m", "--manifest", help="manifest to update")
     parser_add_tag.add_argument("tag", help="tag to add to tags attribute")
     parser_add_tag.set_defaults(func=add_tag)
     if os.path.isfile(MANIFEST_BASENAME):
         parser_add_tag.set_defaults(manifest=MANIFEST_BASENAME)
  
 def _add_subparser_get_value(subparsers):
     parser_get_value = subparsers.add_parser('get-value')
     parser_get_value.add_argument("-m", "--manifest", help="manifest from which to retrieve the value")
     parser_get_value.add_argument("xpath", help="key to print, colon separated for nested values")
     parser_get_value.set_defaults(func=get_value)
     if os.path.isfile(MANIFEST_BASENAME):
         parser_get_value.set_defaults(manifest=MANIFEST_BASENAME)
  
 def _add_subparser_get_first_tag(subparsers):
     parser_get_first_tag = subparsers.add_parser('get-first-tag')
     parser_get_first_tag.add_argument("-m", "--manifest", help="manifest from which to retrieve the first tag")
     parser_get_first_tag.set_defaults(func=get_first_tag)
     if os.path.isfile(MANIFEST_BASENAME):
         parser_get_first_tag.set_defaults(manifest=MANIFEST_BASENAME)
  
 def _add_subparser_list(subparsers):
     parser_list = subparsers.add_parser('list')
     parser_list.add_argument("manifest", help="manifest to list")
     parser_list.add_argument("-i", "--info", action="store_const", const=1, help="include extra info")
     parser_list.add_argument("-t", "--tag", help="tag to list files for")
     parser_list.set_defaults(func=list)
     if os.path.isfile(MANIFEST_BASENAME):
         parser_list.set_defaults(manifest=MANIFEST_BASENAME)
  
 def _add_subparser_clean(subparsers):
     parser_clean = subparsers.add_parser('clean')
     parser_clean.add_argument("-d", "--dry-run", action="store_const", const=1, help="don't actually delete files")
     parser_clean.add_argument("directory", default=".", nargs='?', help="directory to clean (must contain %s)" % MANIFEST_BASENAME)
     parser_clean.add_argument("-e", "--exclude", nargs="+", action='append', help="relative paths to ignore")
     parser_clean.add_argument("-i", "--include", nargs="+", action='append', help="relative paths to include (ignore *)")
     parser_clean.add_argument("-v", "--verbose", action="count", default=0, help="increase output verbosity")
     parser_clean.set_defaults(func=clean)
  
 def _add_subparser_download(subparsers):
     parser_download = subparsers.add_parser('download')
     parser_download.add_argument("-d", "--dest-dir", help="destination directory")
     parser_download.add_argument("-t", "--tag", help="tag to download")
     parser_download.add_argument("-b", "--base-url", default=DEFAULT_BASE_URL, help="base URL")
     parser_download.add_argument("-n", "--name", help="bundle name")
     parser_download.add_argument("--chunk-size", type=int, default=DEFAULT_CHUNK_SIZE, help="download chunk size")
     parser_download.add_argument("-s", "--save-dir", help="save a copy of the manifest files to this directory")
     parser_download.add_argument("-l", "--local-dir", help="directory containing local manifest files")
     parser_download.add_argument("-w", "--wget", default=False, action="store_true", help="use wget to download")
     parser_download.add_argument("-v", "--verbose", action="count", default=0, help="increase output verbosity")
     parser_download.add_argument("files", action="append", nargs="*", default=None, type=str, help="files to download if needed")
     
     parser_download.set_defaults(func=download)
     parser_download.set_defaults(dest_dir=".")
  
 def _add_subparser_generate(subparsers):
     parser_gen = subparsers.add_parser('generate')
     parser_gen.add_argument("-b", "--base-manifest", help="base manifest file")
     parser_gen.add_argument("-c", "--checksum-bytes", default=1000000, help="how many bytes to checksum per file")
     parser_gen.add_argument("-t", "--tag", required=True, help="new tag for manifest")
     parser_gen.add_argument("-f", "--force", action="store_const", const=1, help="generate manifest despite warnings")
     parser_gen.add_argument("-e", "--exclude", nargs="+", action='append', help="relative paths to ignore")
     parser_gen.add_argument("-i", "--include", nargs="+", action='append', help="relative paths to include (ignore *)")
     parser_gen.add_argument("-n", "--name", help="bundle name")
     parser_gen.add_argument("directory", help="directory to generate a manifest for")
     parser_gen.set_defaults(func=generate)
  
 def _add_subparser_list_tags(subparsers):
     parser_list_tags = subparsers.add_parser('list_tags')
     parser_list_tags.add_argument("-b", "--base-url", default=DEFAULT_BASE_URL, help="base URL")
     parser_list_tags.add_argument("--chunk-size", type=int, default=DEFAULT_CHUNK_SIZE, help="download chunk size")
     parser_list_tags.add_argument("-w", "--wget", default=False, action="store_true", help="use wget to download")
     parser_list_tags.add_argument("-v", "--verbose", action="count", default=0, help="increase output verbosity")
     parser_list_tags.set_defaults(func=list_tags)
  
 def create_default_options():
     options = argparse.Namespace(
                 verbose=0,
                 dest_dir=None,
                 tag=None,
                 base_url=DEFAULT_BASE_URL,
                 name=None,
                 chunk_size=DEFAULT_CHUNK_SIZE,
                 save_dir=None,
                 local_dir=None,
                 wget=False,
                 files=None,
                 func=list_tags)
     return options
  
 def run_command(command):
     proc = subprocess.run(command, shell=True)
     if proc.returncode != 0:
         print("Error: return =", proc.returncode, ": trying to run command =", command)
         sys.exit(1)
  
 def reprint(options, args):
     with open(options.manifest, 'rb') as manifest:
         manifest = json.load(manifest)
         print(json.dumps(manifest, indent=4, sort_keys=True))
  
 def update_file(options, args):
     with open(options.manifest, 'rb') as manifest:
         manifest = json.load(manifest)
         current_entry = manifest['files'].get(options.path)
         if os.path.islink(options.path):
             linkValue = os.readlink(options.path)
             if not current_entry or current_entry.get("symlink") != linkValue:
                 info = {"symlink": linkValue, "tag": options.tag}
                 manifest['files'][options.path] = info
         else:
             checksum = _get_checksum(manifest, options.path)
             if not current_entry or current_entry.get('checksum') != checksum:
                 info = {
                     "checksum": checksum, 
                     "size": os.stat(options.path).st_size, 
                     "mode": os.stat(options.path).st_mode, 
                     "tag": manifest['tag']
                 }
                 manifest['files'][options.path] = info
  
         print(json.dumps(manifest, indent=4, sort_keys=True))
  
 def add_tag(options, args):
     with open(options.manifest, 'rb') as manifest:
         manifest = json.load(manifest)
         if options.tag not in manifest["tags"]:
             manifest["tags"].append(options.tag)
         else:
             print("%s is already in the tags attribute" % (options.tag), file=sys.stderr)
  
         print(json.dumps(manifest, indent=4, sort_keys=True))
  
 def get_value(options, args):
     with open(options.manifest, 'rb') as manifest:
         manifest = json.load(manifest)
         for part in options.xpath.split(":"):
             if part in manifest:
                 manifest = manifest[part]
             else:
                 print("Path not found, invalid part: %s" % part)
                 return
         print(manifest)
         
 def get_first_tag(options, args):
     with open(options.manifest, 'rb') as manifest:
         manifest = json.load(manifest)
         print(manifest['tags'][0])
  
 def getFileList(excludeList=None, includeList=None):
     allFiles = []
     
     for root, _, files in os.walk(".", followlinks=True):
         for f in files:
             if '/' in root:
                 name = root[2:]+'/'+f
             else:
                 name = f
     
             # exclude files if not in include list
             addIt = True
             if excludeList:
                 for exclude in excludeList:
                     if exclude[0] == "." or name.startswith(exclude[0]):
                         addIt = False
                         if includeList:
                             for include in includeList:
                                 if name.startswith(include[0]):
                                     addIt = True
                                     break
                         if not addIt:
                             break
             if addIt:
                 if "__pycache__" not in name:
                     allFiles.append(name)
     
     return allFiles
  
 def clean(options, args):
     os.chdir(options.directory)
  
     # check for exclude wild card
     if options.exclude:
         for exclude in options.exclude:
             if exclude[0] == ".":
                 return
     
     if not os.path.isfile(MANIFEST_BASENAME):
         print("directory needs to contain a", MANIFEST_BASENAME)
         return 1
  
     with open(MANIFEST_BASENAME, 'rb') as manifest:
         manifest = json.load(manifest)
         files = manifest["files"]
         for f in getFileList(options.exclude, options.include):
             if f == MANIFEST_BASENAME:
                 continue
             if not files.get(f):
                 if options.verbose or options.dry_run:
                     print("cleaning %s" % (f))
                 if not options.dry_run:
                     try:
                         os.remove(f)
                     except FileNotFoundError:
                         pass # just ignore the file if it does not exist
  
 def list(options, args):
     if os.path.isdir(options.manifest):
         options.manifest = "%s/%s" % (options.manifest, MANIFEST_BASENAME)
     with open(options.manifest, 'rb') as manifest:
         manifest = json.load(manifest)
         if options.info:
             for f, info in manifest["files"].items():
                 if not options.tag or info["tag"] == options.tag:
                     if info.get('symlink'):
                         print("%s %s, -> %s" % (f, info["tag"], info["symlink"]))
                     else:
                         print("%s %s, %s bytes, %s" % (f, info["tag"], info["size"], info["checksum"]))
         elif options.tag:
             for f, info in manifest["files"].items():
                 if info["tag"] == options.tag:
                     print(f)
         else:
             for f in manifest["files"]:
                 print(f)
  
 def generate(options, args):
     if not options.base_manifest and os.path.isfile("%s/%s" % (options.directory, MANIFEST_BASENAME)):
         options.base_manifest = "%s/%s" % (options.directory, MANIFEST_BASENAME)
  
     manifest = None
     if options.base_manifest and os.path.isfile(options.base_manifest) and os.path.getsize(options.base_manifest):
         with open(options.base_manifest, 'rb') as base_manifest:
             manifest = json.load(base_manifest)
     else:
         manifest = {"checksum_bytes": options.checksum_bytes, "tags": []}
  
     manifest["tags"] = [options.tag]
  
     os.chdir(options.directory)
  
     all_files = getFileList(options.exclude, options.include)
  
     if options.name:
         manifest['name'] = options.name
  
     files_entries = manifest.get("files", {})
  
     # delete entries not in the directory
     files_to_delete = []
     if "files" in manifest:
         for path, info in manifest["files"].items():
             if path not in all_files:
                 files_to_delete.append(path)
     for path in files_to_delete:
         del files_entries[path]
  
     for f in all_files:
         if os.path.basename(f) == MANIFEST_BASENAME:
             continue
  
         current_entry = files_entries.get(f)
         if os.path.islink(f):
             linkValue = os.readlink(f)
             if not current_entry or current_entry.get("symlink") != linkValue:
                 info = {"symlink": linkValue, "tag": options.tag}
                 files_entries[f] = info
         else:
             fileSize = os.path.getsize(f)
             checksum = _get_checksum(manifest, f)
             if not current_entry or current_entry.get('size') != fileSize or current_entry.get('checksum') != checksum:
                 info = {
                     "checksum": checksum, 
                     "size": fileSize, 
                     "mode": os.stat(f).st_mode, 
                     "tag": options.tag
                 }
                 files_entries[f] = info
     manifest["files"] = files_entries
     print(json.dumps(manifest, indent=4, sort_keys=True))
  
 def download(options, args):
     manifest = None
     manifest_filename = "%s/%s" % (options.dest_dir, MANIFEST_BASENAME)
  
     if not os.path.isdir(options.dest_dir):
         os.makedirs(options.dest_dir)
  
     if options.local_dir:
         if options.save_dir:
             print("Error: Can not have --local_dir and --save_dir")
             return 1
  
     if not options.tag or not options.name:
         if not os.path.isfile(manifest_filename):
             print("must have -t and -n or %s" % (manifest_filename))
             return 1
         with open(manifest_filename, 'rb') as manifest:
             manifest = json.load(manifest)
         if not options.tag:
             options.tag = manifest['tags'][-1]
         if not options.name:
             options.name = manifest['name']
  
     if not _download_file(options, MANIFEST_BASENAME):
         return 1
  
     with open(manifest_filename, 'rb') as manifest:
         manifest = json.load(manifest)
  
     modified_files = _check_directory_against_manifest(options, options.dest_dir, manifest)
  
     # if files on command line only look at those
     if options.files and options.files[0]:
         newList = {}
         for f in  options.files[0]:
             try:
                 newList[f] = modified_files[f]
             except:
                 pass
         modified_files = newList
  
     if not modified_files:
         if options.verbose:
             print("No files require downloading")
     else:
         _download_files(options, modified_files)
  
     if options.save_dir:
         for path, info in manifest['files'].items():
             if info.get('checksum'):
                 src = "%s/%s" % (options.dest_dir, path)
                 dest = "%s/%s/%s/%s" % (options.save_dir, info["tag"], options.name, path)
                 destDir = os.path.dirname(dest)
                 if not os.path.isdir(destDir):
                     os.makedirs(destDir)
                 shutil.copy(src, dest)
                 os.chmod(dest, info["mode"])
  
 def get_tags(options, args):
     tag_list = []
     tempDir = tempfile.TemporaryDirectory(prefix="manifest-")
     status = 0
     url = options.base_url + "/"
     if options.wget:
         command = "cd %s; wget -q %s" % (tempDir.name, url)
         run_command(command)
     else:
         parts = urllib.parse.urlparse(url)
         host = parts.netloc
         request = parts.path
         status = httpdl(host, request, localpath=tempDir.name, 
                         outputfilename="index.html",
                         verbose=options.verbose,
                         force_download=True,
                         chunk_size=options.chunk_size)
     if status == 0:
         with open("%s/index.html" % (tempDir.name)) as f: 
             inBody = False
             for line in f:
                 if "<body>" in line:
                     inBody = True
                 if "</body>" in line:
                     break
                 if inBody:
                     if line.startswith("<a href="): 
                         parts = line.split('"') 
                         s = parts[1].split("/")[0] 
                         if s != "..": 
                             tag_list.append(s)
         return tag_list
     else:
         print("Error downloading list of tags : return code =", status)
         return tag_list
  
 def list_tags(options, args):
     for tag in get_tags(options, args):
         print(tag)
  
 def check_tag(options, args):
     for tag in get_tags(options, args):
         if tag == options.tag:
             return True
     return False
  
 def _get_checksum(manifest, path):
     checksum = hashlib.sha256()
     with open(path, 'rb') as current_file:
         checksum.update(current_file.read(manifest['checksum_bytes']))
     return checksum.hexdigest()
  
 def _check_directory_against_manifest(options, directory, manifest):
     modified_files = {}
     for path, info in manifest['files'].items():
         dest = os.path.join(directory, path)
         if os.path.islink(dest):
             if info.get('symlink') != os.readlink(dest):
                 modified_files[path] = info
         elif os.path.isfile(dest):
             if info.get('size') != os.path.getsize(dest) or info.get('checksum') != _get_checksum(manifest, dest) or info.get('mode') != os.stat(dest).st_mode:
                 modified_files[path] = info
         else:
             modified_files[path] = info
     return modified_files
  
 def _download_file(options, fileName):
     dest = "%s/%s" % (options.dest_dir, fileName)
     dest_dir = os.path.dirname(dest)
     if not os.path.isdir(dest_dir):
         os.makedirs(dest_dir)
     
     if options.local_dir:
         src = "%s/%s/%s/%s" % (options.local_dir, options.tag, options.name, fileName)
         if options.verbose:
             print("Copying %s from %s" % (fileName, src))
         shutil.copy(src, dest)
         return True
     
     url = "%s/%s/%s/%s" % (options.base_url, options.tag, options.name, fileName)
     if options.verbose:
         print("Downloading %s from %s" % (fileName, url))
     if options.wget:
         if os.path.isfile(dest):
             os.remove(dest)
         command = "cd %s; wget -q %s" % (dest_dir, url)
         run_command(command)
     else:
         parts = urllib.parse.urlparse(url)
         #host = "%s://%s" % (parts.scheme, parts.netloc)
         host = parts.netloc
         request = parts.path
         status = httpdl(host, request, localpath=dest_dir, 
                         outputfilename=os.path.basename(dest),
                         verbose=options.verbose,
                         force_download=True,
                         chunk_size=options.chunk_size)
         if status != 0:
             print("Error downloading", dest, ": return code =", status)
             return False
  
     if options.save_dir:
         src = "%s/%s" % (options.dest_dir, fileName)
         dest = "%s/%s/%s/%s" % (options.save_dir, options.tag, options.name, fileName)
         destDir = os.path.dirname(dest)
         if not os.path.isdir(destDir):
             os.makedirs(destDir)
         shutil.copy(src, dest)
     return True
  
 def _download_files(options, file_list):
     if options.local_dir:
         for path, info in file_list.items():
             dest = "%s/%s" % (options.dest_dir, path)
             dest_dir = os.path.dirname(dest)
             if not os.path.isdir(dest_dir):
                 os.makedirs(dest_dir)
             if info.get('checksum'):
                 src = "%s/%s/%s/%s" % (options.local_dir, info["tag"], options.name, path)
                 shutil.copy(src, dest)
                 os.chmod(dest, info["mode"])
             else:
                 src = info['symlink']
                 os.symlink(src, dest)
         return
         
     if options.wget:
         if not os.path.isdir(options.dest_dir):
             os.makedirs(options.dest_dir)
         with tempfile.NamedTemporaryFile(prefix="manifest-") as txt_file:
             for path, info in file_list.items():
                 if info.get('checksum'):
                     txt_file.write("%s\n" % path)
                 else:
                     dest = "%s/%s" % (options.dest_dir, path)
                     src = info['symlink']
                     os.symlink(src, dest)
             command = "cd %s; wget -x -nH -i %s --cut-dirs=3 --base=%s/%s/%s/" % (options.dest_dir, txt_file.name, options.base_url, info["tag"], options.name)
             run_command(command)
         for path, info in file_list.items():
             if info.get('checksum'):
                 os.chmod(dest, info["mode"])
         return
  
     for path, info in file_list.items():
         dest = "%s/%s" % (options.dest_dir, path)
         dest_dir = os.path.dirname(dest)
         if not os.path.isdir(dest_dir):
             os.makedirs(dest_dir)
  
         if info.get('checksum'):
             if os.path.islink(dest) or os.path.exists(dest):
                 os.remove(dest)
             url = "%s/%s/%s/%s" % (options.base_url, info["tag"], options.name, path)
             if options.verbose:
                 print("Downloading %s from %s" % (path, url))
             parts = urllib.parse.urlparse(url)
             host = parts.netloc
             request = parts.path
             status = httpdl(host, request, localpath=dest_dir, 
                         outputfilename=os.path.basename(dest),
                         verbose=options.verbose,
                         force_download=True,
                         chunk_size=options.chunk_size)
             if status == 0:
                 os.chmod(dest, info["mode"])
             else:
                 print("Error downloading", dest, ": return code =", status)
         else:
             src = info['symlink']
             if options.verbose:
                 print("Making symlink %s -> %s" % (dest, src))
             if os.path.islink(dest) or os.path.exists(dest):
                 os.remove(dest)
             os.symlink(src, dest)
  
  
 if __name__ == "__main__":
     sys.exit(run())
manifest.getSession
def getSession(verbose=0, ntries=5)
Definition: manifest.py:45
manifest.get_tags
def get_tags(options, args)
Definition: manifest.py:556
manifest.run
def run()
Definition: manifest.py:181
manifest.reprint
def reprint(options, args)
Definition: manifest.py:313
manifest.run_command
def run_command(command)
Definition: manifest.py:307
manifest.list
def list(options, args)
Definition: manifest.py:423
void print(std::ostream &stream, const char *format)
Definition: PrintDebug.hpp:38
manifest.download
def download(options, args)
Definition: manifest.py:498
manifest.generate
def generate(options, args)
Definition: manifest.py:443
manifest.clean
def clean(options, args)
Definition: manifest.py:395
manifest.list_tags
def list_tags(options, args)
Definition: manifest.py:592
manifest.create_default_options
def create_default_options()
Definition: manifest.py:292
manifest.update_file
def update_file(options, args)
Definition: manifest.py:318
manifest.add_tag
def add_tag(options, args)
Definition: manifest.py:340
manifest.uncompressFile
def uncompressFile(compressed_file)
Definition: manifest.py:149
manifest.get_file_time
def get_file_time(localFile)
Definition: manifest.py:171
manifest.get_first_tag
def get_first_tag(options, args)
Definition: manifest.py:361
manifest.get_value
def get_value(options, args)
Definition: manifest.py:350
manifest.isRequestAuthFailure
def isRequestAuthFailure(req)
Definition: manifest.py:66
manifest.check_tag
def check_tag(options, args)
Definition: manifest.py:596
manifest.httpdl
def httpdl(server, request, localpath='.', outputfilename=None, ntries=5, uncompress=False, timeout=30., verbose=0, force_download=False, chunk_size=DEFAULT_CHUNK_SIZE)
Definition: manifest.py:75
manifest.getFileList
def getFileList(excludeList=None, includeList=None)
Definition: manifest.py:366