A sphere with three ocean waves in differing shades of blue. Next to the sphere, there is the bolded text "Ocean Color". Under that, there are two acronyms, separated by a vertical pipe: OB.DAAC (Ocean Biology Distributed Active Archive Center) and OBPG (Ocean Biology Processing Group). Lorem ipsum dolor sit amet
NASA Logo
Ocean Color Science Software

ocssw V2022
manifest_ocssw.py
Go to the documentation of this file.
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 """
4 Created on Wed Aug 22 15:11:27 2018
5 
6 @author: dshea
7 """
8 
9 import sys
10 import argparse
11 import os
12 import hashlib
13 import json
14 import subprocess
15 import shutil
16 
17 
18 MANIFEST_BASENAME = "manifest.json"
19 BUNDLELIST_BASENAME = "bundleList.json"
20 
21 # configInfo contains global info about the ocssw_manifest
22 # baseDir - base directory holding the manifest bundles
23 # tag - current installed tag
24 configInfo = {}
25 configInfoFilename = ".manifest_ocssw.json"
26 startingDir = ""
27 
28 manifestCommand = os.path.dirname(__file__) + "/manifest.py"
29 manifestFilename = "manifest.json"
30 
31 remoteHosts = ["ocssw@gs616-container106"]
32 remoteRepoDir = "/data1/manifest/tags"
33 
34 
38 initialBundleList = [
39  {"name":"root", "dir":".", "help":"random files in the root dir", "extra":"--exclude . --include bundleList.json --include OCSSW_bash.env --include OCSSW.env", "commandLine":True},
40  {"name":"python", "dir":"python", "help":"OCSSW required python modules", "commandLine":True},
41  {"name":"bin_linux_64", "dir":"bin_linux_64", "help":"executables for Linux", "commandLine":False},
42  {"name":"bin_linux_hpc", "dir":"bin_linux_hpc", "help":"executables for Linux HPC", "commandLine":False},
43  {"name":"bin_macosx_intel", "dir":"bin_macosx_intel", "help":"executables for Mac", "commandLine":False},
44  {"name":"bin_macosx_arm64", "dir":"bin_macosx_arm64", "help":"executables for Mac ARM64", "commandLine":False},
45  {"name":"bin_odps", "dir":"bin_odps", "help":"executables for ODPS", "commandLine":False},
46  {"name":"lib_linux_64", "dir":"lib_linux_64", "help":"shared libraries for Linux", "commandLine":False},
47  {"name":"lib_linux_hpc", "dir":"lib_linux_hpc", "help":"shared libraries for Linux HPC", "commandLine":False},
48  {"name":"lib_macosx_intel", "dir":"lib_macosx_intel", "help":"shared libraries for Mac", "commandLine":False},
49  {"name":"lib_macosx_arm64", "dir":"lib_macosx_arm64", "help":"shared libraries for Mac ARM64", "commandLine":False},
50  {"name":"lib_odps", "dir":"lib_odps", "help":"shared libraries for ODPS", "commandLine":False},
51  {"name":"opt_linux_64", "dir":"opt_linux_64", "help":"3rd party library for Linux", "extra": "--exclude src", "commandLine":False},
52  {"name":"opt_linux_hpc", "dir":"opt_linux_hpc", "help":"3rd party library for Linux HPC", "extra": "--exclude src", "commandLine":False},
53  {"name":"opt_macosx_intel", "dir":"opt_macosx_intel", "help":"3rd party library for Mac", "extra": "--exclude src", "commandLine":False},
54  {"name":"opt_macosx_arm64", "dir":"opt_macosx_arm64", "help":"3rd party library for Mac ARM64", "extra": "--exclude src", "commandLine":False},
55  {"name":"opt_odps", "dir":"opt_odps", "help":"3rd party library for ODPS", "extra": "--exclude src", "commandLine":False},
56 
57  {"name":"ocssw_src", "dir":"ocssw_src", "help":"OCSSW source code", "commandLine":False},
58  {"name":"opt_src", "dir":"opt/src", "help":"3rd party library sources", "extra":"--exclude .buildit.db", "commandLine":True},
59 
60  {"name":"afrt", "dir":"share/afrt", "help":"Ahmad-Fraser RT data", "commandLine":True},
61  {"name":"aquaverse", "dir":"share/aquaverse", "help":"Algorithm based on Mixture Density Networks", "commandLine":True},
62  {"name":"avhrr", "dir":"share/avhrr", "help":"AVHRR", "commandLine":True},
63  {"name":"aviris", "dir":"share/aviris", "help":"AVIRIS", "commandLine":True},
64  {"name":"common", "dir":"share/common", "help":"common", "commandLine":True},
65  {"name":"czcs", "dir":"share/czcs", "help":"CZCS", "commandLine":True},
66  {"name":"eval", "dir":"share/eval", "help":"evaluation", "commandLine":True},
67  {"name":"goci", "dir":"share/goci", "help":"GOCI", "commandLine":True},
68  {"name":"harp2", "dir":"share/harp2", "help":"HARP 2 Polarimeter", "commandLine":True},
69  {"name":"hawkeye", "dir":"share/hawkeye", "help":"Hawkeye", "commandLine":True},
70  {"name":"hico", "dir":"share/hico", "help":"HICO", "commandLine":True},
71  {"name":"l5tm", "dir":"share/l5tm", "help":"l5tm", "commandLine":True},
72  {"name":"l7etmp", "dir":"share/l7etmp", "help":"l7etmp", "commandLine":True},
73  {"name":"meris", "dir":"share/meris", "help":"MERIS", "commandLine":True},
74  {"name":"misr", "dir":"share/misr", "help":"MISR", "commandLine":True},
75  {"name":"modis", "dir":"share/modis", "help":"MODIS common", "extra":"--exclude aqua --exclude terra", "commandLine":False},
76  {"name":"modisa", "dir":"share/modis/aqua", "help":"MODIS AQUA", "commandLine":True},
77  {"name":"modist", "dir":"share/modis/terra", "help":"MODIS TERRA", "commandLine":True},
78  {"name":"msi", "dir":"share/msi", "help":"MSI Sentinel 2 common", "extra":"--exclude s2a --exclude s2b", "commandLine":False},
79  {"name":"msis2a", "dir":"share/msi/s2a", "help":"MSI Sentinel 2A", "commandLine":True},
80  {"name":"msis2b", "dir":"share/msi/s2b", "help":"MSI Sentinel 2B", "commandLine":True},
81  {"name":"oci", "dir":"share/oci", "help":"PACE OCI", "commandLine":True},
82  {"name":"ocm1", "dir":"share/ocm1", "help":"OCM1", "commandLine":True},
83  {"name":"ocm2", "dir":"share/ocm2", "help":"OCM2", "commandLine":True},
84  {"name":"ocrvc", "dir":"share/ocrvc", "help":"OC Virtual Constellation", "commandLine":True},
85  {"name":"octs", "dir":"share/octs", "help":"OCTS", "commandLine":True},
86  {"name":"olci", "dir":"share/olci", "help":"OLCI Sentinel 3 common", "extra":"--exclude s3a --exclude s3b", "commandLine":False},
87  {"name":"olcis3a", "dir":"share/olci/s3a", "help":"OLCI Sentinel 3A", "commandLine":True},
88  {"name":"olcis3b", "dir":"share/olci/s3b", "help":"OLCI Sentinel 3B", "commandLine":True},
89  {"name":"oli", "dir":"share/oli", "help":"OLI Landsat", "extra":"--exclude l8 --exclude l9", "commandLine":False},
90  {"name":"olil8", "dir":"share/oli/l8", "help":"OLI Landsat 8", "commandLine":True},
91  {"name":"olil9", "dir":"share/oli/l9", "help":"OLI Landsat 9", "commandLine":True},
92  {"name":"prism", "dir":"share/prism", "help":"PRISM", "commandLine":True},
93  {"name":"sabiamar", "dir":"share/sabiamar", "help":"Sabiamar", "commandLine":True},
94  {"name":"seawifs", "dir":"share/seawifs", "help":"SeaWiFS", "commandLine":True},
95  {"name":"sgli", "dir":"share/sgli", "help":"SGLI", "commandLine":True},
96  {"name":"spexone", "dir":"share/spexone", "help":"SPEX One Polarimeter", "commandLine":True},
97  {"name":"viirs", "dir":"share/viirs", "extra":"--exclude dem --exclude j1 --exclude j2 --exclude npp", "help":"VIIRS common", "commandLine":False},
98  {"name":"viirsdem", "dir":"share/viirs/dem", "help":"VIIRS Digital Elevation", "commandLine":True},
99  {"name":"viirsj1", "dir":"share/viirs/j1", "help":"VIIRS JPSS1", "commandLine":True},
100  {"name":"viirsj2", "dir":"share/viirs/j2", "help":"VIIRS JPSS2", "commandLine":True},
101  {"name":"viirsn", "dir":"share/viirs/npp", "help":"VIIRS NPP", "commandLine":True},
102  {"name":"wv3", "dir":"share/wv3", "help":"WV3", "commandLine":True},
103  {"name":"aerosol", "dir":"share/aerosol", "help":"aerosol processing with dtdb", "commandLine":True},
104  {"name":"cloud", "dir":"share/cloud", "help":"cloud properties processing", "commandLine":True},
105  {"name":"telemetery", "dir":"share/telemetry", "help":"telemetry packet descriptions", "commandLine":True},
106  {"name":"benchmark", "dir":"benchmark", "help":"benchmark MOSIS Aqua, level0 -> level3 Mapped", "commandLine":True},
107  {"name":"viirs_l1_benchmark", "dir":"viirs_l1_benchmark", "help":"VIIRS benchmark data", "commandLine":True},
108  {"name":"viirs_l1_bin_macosx_intel", "dir":"viirs_l1_bin_macosx_intel", "help":"Subset of binary files for VIIRS", "commandLine":False},
109  {"name":"viirs_l1_bin_macosx_arm64", "dir":"viirs_l1_bin_macosx_arm64", "help":"Subset of binary files for VIIRS", "commandLine":False},
110  {"name":"viirs_l1_bin_linux_64", "dir":"viirs_l1_bin_linux_64", "help":"Subset of binary files for VIIRS", "commandLine":False},
111  {"name":"viirs_l1_bin_odps", "dir":"viirs_l1_bin_odps", "help":"Subset of binary files for VIIRS", "commandLine":False}
112 ]
113 
114 def add_subparser_init_dir(subparsers):
115  newParser = subparsers.add_parser('init', help="Initialize the directory to be the root of the OCSSW manifest bundles")
116  newParser.add_argument("dir", help="directory to initialize", nargs='?', default='.')
117  newParser.set_defaults(func=init_dir)
118 
119 def add_subparser_diff(subparsers):
120  newParser = subparsers.add_parser('diff', help="Show the difference between the old and new text files")
121  newParser.add_argument("tag", nargs='?', help="tag name to use")
122  newParser.set_defaults(func=diff)
123  newParser.set_defaults(difftool=None)
124 
125 def add_subparser_difftool(subparsers):
126  newParser = subparsers.add_parser('difftool', help="Show the difference between the old and new text files using diff tool such as meld")
127  newParser.add_argument("difftool", nargs='?', help="tool name to use for diff")
128  newParser.add_argument("tag", nargs='?', help="tag name to use")
129  newParser.set_defaults(func=diff)
130 
131 def add_subparser_status(subparsers):
132  newParser = subparsers.add_parser('status', help="List the files that differ from the manifest")
133  newParser.add_argument("tag", nargs='?', help="tag name to use")
134  newParser.set_defaults(func=status)
135 
136 def add_subparser_push(subparsers):
137  newParser = subparsers.add_parser('push', help="Create a new tag and push to the server")
138  newParser.add_argument("tag", help="tag name to use")
139  newParser.add_argument("-o", "--overwrite", default=False, action="store_true", help="overwrite the tag if it already exists")
140  newParser.set_defaults(func=push)
141 
142 def add_subparser_pull(subparsers):
143  newParser = subparsers.add_parser('pull', help="Download a tag from the server")
144  newParser.add_argument("tag", help="tag name to use")
145  newParser.add_argument("-c", "--clean", default=False, action="store_true", help="clean the directory after downloading")
146  newParser.set_defaults(func=pull)
147 
148 def add_subparser_list(subparsers):
149  newParser = subparsers.add_parser('list', help="List the tags on the remote system")
150  newParser.set_defaults(func=list_tags)
151 
152 
153 def init_dir(options):
154  global configInfo
155 
156  if os.path.exists(options.dir):
157  if not os.path.isdir(options.dir):
158  print("Error:", options.dir, "exists and is not a directory")
159  else:
160  os.mkdir(options.dir)
161 
162  fileName = os.path.join(os.path.abspath(options.dir), configInfoFilename)
163  if os.path.exists(fileName):
164  print("Error: manifest_ocssw config file", fileName, "already exists")
165  sys.exit(1)
166 
167  # fill up an initial config structure
168  configInfo["baseDir"] = os.path.abspath(options.dir)
169  configInfo["tag"] = "Unknown"
170 
171  # save as a JSON file
172  with open(fileName, 'w') as outfile:
173  json.dump(configInfo, outfile, indent=4, sort_keys=True)
174 
175  fileName = os.path.join(os.path.abspath(options.dir), "bundleList.json")
176  with open(fileName, 'w') as outfile:
177  json.dump(initialBundleList, outfile, indent=4, sort_keys=True)
178 
179 
180 def runCommand(command, pipe=False, shellVal=False):
181  if pipe:
182  proc = subprocess.run(command, stdout=subprocess.PIPE, shell=shellVal)
183  else:
184  proc = subprocess.run(command, shell=shellVal)
185  if proc.returncode != 0:
186  print("Error: trying to run = ", command)
187  sys.exit(1)
188  if pipe:
189  return proc.stdout
190 
191 
192 def tagExists(tag):
193  command = ["ssh", "-q", remoteHosts[0], "[ -d %s/%s ]" % (remoteRepoDir, tag) ]
194  proc = subprocess.run(command)
195  if proc.returncode == 0:
196  return True
197  else:
198  return False
199 
200 
201 def bundleExists(tag, name):
202  command = ["ssh", "-q", remoteHosts[0], "[ -d %s/%s/%s ]" % (remoteRepoDir, tag, name) ]
203  proc = subprocess.run(command)
204  if proc.returncode == 0:
205  return True
206  else:
207  return False
208 
209 def findBundleInList(name, bundleList):
210  if bundleList:
211  for info in bundleList:
212  if info["name"] == name:
213  return info
214  return None
215 
216 def diff(options, bundleInfo):
217  global startingDir
218 
219  # only do a status if user is sitting in the bundel directory
220  bundleDir = os.path.abspath(os.path.join(configInfo["baseDir"], bundleInfo["dir"]))
221  if startingDir not in bundleDir:
222  return
223 
224  print(bundleInfo["name"], end=' ', flush=True)
225 
226  statusTag = "tempStatusTag"
227  os.chdir(configInfo["baseDir"])
228 
229  currentManifest = {}
230  if hasattr(options, "tag") and options.tag:
231  if tagExists(options.tag):
232  if bundleExists(options.tag, bundleInfo["name"]):
233  command = ["scp", "-q", "%s:%s/%s/%s/%s" % (remoteHosts[0], remoteRepoDir, options.tag, bundleInfo["name"], manifestFilename), "/tmp"]
234  runCommand(command)
235  with open("/tmp/" + manifestFilename, 'rb') as manifest:
236  currentManifest = json.load(manifest)
237  os.remove("/tmp/" + manifestFilename)
238  else:
239  print("Warning: bundle %s does not exist in tag %s on server." % (bundleInfo["name"], options.tag))
240  else:
241  print("Error: tag:", options.tag, "does not exist on server")
242  sys.exit(1)
243  else:
244  # open current manifest
245  if bundleExists(configInfo["tag"], bundleInfo["name"]):
246  command = ["scp", "-q", "%s:%s/%s/%s/%s" % (remoteHosts[0], remoteRepoDir, configInfo["tag"], bundleInfo["name"], manifestFilename), "/tmp"]
247  runCommand(command)
248  with open("/tmp/" + manifestFilename, 'rb') as manifest:
249  currentManifest = json.load(manifest)
250  os.remove("/tmp/" + manifestFilename)
251 
252  if not os.path.isdir(bundleInfo["dir"]):
253  print("Warning: bundle directory %s does not exist." % (bundleInfo["dir"]))
254  return
255 
256  command = [manifestCommand, "generate",
257  "-n", bundleInfo["name"],
258  "-t", statusTag,
259  bundleInfo["dir"]]
260  if "extra" in bundleInfo:
261  command += bundleInfo["extra"].split()
262  statusManifest = json.loads(runCommand(command, pipe=True))
263 
264  if "tags" in currentManifest:
265  tagList = currentManifest["tags"]
266  versionStr = "("
267  if len(tagList) > 1:
268  versionStr += tagList[0] + ".."
269  versionStr += tagList[-1] + ")"
270  print(versionStr)
271  else:
272  print()
273 
274  # list new files
275  fileList = []
276  for f, info in statusManifest["files"].items():
277  if ("files" not in currentManifest) or (f not in currentManifest["files"]):
278  fileList.append(f)
279  if fileList:
280  print(" New Files:")
281  for f in fileList:
282  print(" %s/%s" % (bundleInfo["dir"], f))
283 
284  # list modified files
285  fileList = []
286  for f, info in statusManifest["files"].items():
287  if ("files" in currentManifest) and (f in currentManifest["files"]):
288  if "symlink" in currentManifest["files"][f]:
289  # both symlink
290  if "symlink" in info:
291  if info["symlink"] != currentManifest["files"][f]["symlink"]:
292  fileList.append(f)
293  # remote symlink
294  else:
295  fileList.append(f)
296  # local symlink
297  elif "symlink" in info:
298  fileList.append(f)
299  elif info["tag"] != currentManifest["files"][f]["tag"]:
300  localFilename = "%s/%s" % (bundleInfo["dir"], f)
301  with open(localFilename, "rb") as localFile:
302  bytes = localFile.read()
303  localFileChecksum = hashlib.sha256(bytes).hexdigest()
304  if "checksum" in currentManifest["files"][f]:
305  if localFileChecksum != currentManifest["files"][f]["checksum"]:
306  fileList.append(f)
307 
308  if fileList:
309  print(" Modified Files:")
310  for f in fileList:
311  print(" %s/%s" % (bundleInfo["dir"], f))
312  if "symlink" in currentManifest["files"][f]:
313  # both symlink
314  if "symlink" in statusManifest["files"][f]:
315  print("diff %s remote..local" % (f))
316  print("< " + currentManifest["files"][f]["tag"] + " = " + currentManifest["files"][f]["symlink"])
317  print("---")
318  print("> current dir = " + statusManifest["files"][f]["symlink"])
319  # remot symlink
320  else:
321  print("diff %s remote..local" % (f))
322  print("< " + currentManifest["files"][f]["tag"] + " = " + currentManifest["files"][f]["symlink"])
323  print("---")
324  print("> current dir = Not a symlink")
325  # local symlink
326  elif "symlink" in statusManifest["files"][f]:
327  print("diff %s remote..local" % (f))
328  print("< " + currentManifest["files"][f]["tag"] + " = Not a symlink")
329  print("---")
330  print("> current dir = " + statusManifest["files"][f]["symlink"])
331  # neither symlink
332  else:
333  try:
334  localFilename = "%s/%s" % (bundleInfo["dir"], f)
335  subproc = subprocess.run(["file", '-L', localFilename], capture_output=True, text=True, shell=False)
336  if "ASCII" in subproc.stdout:
337  remoteFilename = "%s/%s" % ("/tmp/", os.path.basename(f))
338  file_tag = currentManifest["files"][f]["tag"]
339  command = ["scp", "-q", "%s:%s/%s/%s/%s" % (remoteHosts[0], remoteRepoDir, file_tag, bundleInfo["name"], f), "/tmp"]
340  status = subprocess.run(command, shell=False).returncode
341  if status == 0:
342  if options.difftool is not None:
343  command = [options.difftool, remoteFilename, localFilename]
344  subprocess.run(command, shell=False)
345  else:
346  print("diff %s remote..local" % (f))
347  command = ["diff",remoteFilename, localFilename]
348  subprocess.run(command, shell=False)
349  os.remove(remoteFilename)
350  except:
351  continue
352 
353 
354 
355  # deleted file
356  fileList = []
357  if "files" in currentManifest:
358  for f, info in currentManifest["files"].items():
359  if f not in statusManifest["files"]:
360  fileList.append(f)
361  if fileList:
362  print(" Deleted Files:")
363  for f in fileList:
364  print(" %s/%s" % (bundleInfo["dir"], f))
365 
366 def status(options, bundleInfo):
367  global startingDir
368 
369  # only do a status if user is sitting in the bundel directory
370  bundleDir = os.path.abspath(os.path.join(configInfo["baseDir"], bundleInfo["dir"]))
371  if startingDir not in bundleDir:
372  return
373 
374  print(bundleInfo["name"], end=' ', flush=True)
375 
376  statusTag = "tempStatusTag"
377  os.chdir(configInfo["baseDir"])
378 
379  currentManifest = {}
380  if hasattr(options, "tag") and options.tag:
381  if tagExists(options.tag):
382  if bundleExists(options.tag, bundleInfo["name"]):
383  command = ["scp", "-q", "%s:%s/%s/%s/%s" % (remoteHosts[0], remoteRepoDir, options.tag, bundleInfo["name"], manifestFilename), "/tmp"]
384  runCommand(command)
385  with open("/tmp/" + manifestFilename, 'rb') as manifest:
386  currentManifest = json.load(manifest)
387  os.remove("/tmp/" + manifestFilename)
388  else:
389  print("Warning: bundle %s does not exist in tag %s on server." % (bundleInfo["name"], options.tag))
390  else:
391  print("Error: tag:", options.tag, "does not exist on server")
392  sys.exit(1)
393  else:
394  # open current manifest
395  if bundleExists(configInfo["tag"], bundleInfo["name"]):
396  command = ["scp", "-q", "%s:%s/%s/%s/%s" % (remoteHosts[0], remoteRepoDir, configInfo["tag"], bundleInfo["name"], manifestFilename), "/tmp"]
397  runCommand(command)
398  with open("/tmp/" + manifestFilename, 'rb') as manifest:
399  currentManifest = json.load(manifest)
400  os.remove("/tmp/" + manifestFilename)
401 
402  if not os.path.isdir(bundleInfo["dir"]):
403  print("Warning: bundle directory %s does not exist." % (bundleInfo["dir"]))
404  return
405 
406  command = [manifestCommand, "generate",
407  "-n", bundleInfo["name"],
408  "-t", statusTag,
409  bundleInfo["dir"]]
410  if "extra" in bundleInfo:
411  command += bundleInfo["extra"].split()
412  statusManifest = json.loads(runCommand(command, pipe=True))
413 
414  if "tags" in currentManifest:
415  tagList = currentManifest["tags"]
416  versionStr = "("
417  if len(tagList) > 1:
418  versionStr += tagList[0] + ".."
419  versionStr += tagList[-1] + ")"
420  print(versionStr)
421  else:
422  print()
423 
424  # list new files
425  fileList = []
426  for f, info in statusManifest["files"].items():
427  if ("files" not in currentManifest) or (f not in currentManifest["files"]):
428  fileList.append(f)
429  if fileList:
430  print(" New Files:")
431  for f in fileList:
432  print(" %s/%s" % (bundleInfo["dir"], f))
433 
434  # list modified files
435  fileList = []
436  for f, info in statusManifest["files"].items():
437  if ("files" in currentManifest) and (f in currentManifest["files"]):
438  if "symlink" in currentManifest["files"][f]:
439  # both symlink
440  if "symlink" in info:
441  if info["symlink"] != currentManifest["files"][f]["symlink"]:
442  fileList.append(f)
443  # remote symlink
444  else:
445  fileList.append(f)
446  # local symlink
447  elif "symlink" in info:
448  fileList.append(f)
449  # neither symlink
450  elif info["tag"] != currentManifest["files"][f]["tag"]:
451  localFilename = "%s/%s" % (bundleInfo["dir"], f)
452  with open(localFilename, "rb") as localFile:
453  bytes = localFile.read()
454  localFileChecksum = hashlib.sha256(bytes).hexdigest()
455  if "checksum" in currentManifest["files"][f]:
456  if localFileChecksum != currentManifest["files"][f]["checksum"]:
457  fileList.append(f)
458  if fileList:
459  print(" Modified Files:")
460  for f in fileList:
461  print(" %s/%s" % (bundleInfo["dir"], f))
462 
463  # deleted file
464  fileList = []
465  if "files" in currentManifest:
466  for f, info in currentManifest["files"].items():
467  if f not in statusManifest["files"]:
468  fileList.append(f)
469  if fileList:
470  print(" Deleted Files:")
471  for f in fileList:
472  print(" %s/%s" % (bundleInfo["dir"], f))
473 
474 def push(options, bundleInfo):
475  os.chdir(configInfo["baseDir"])
476  command = [manifestCommand, "generate",
477  "-n", bundleInfo["name"],
478  "-t", options.tag,
479  bundleInfo["dir"]]
480  if "extra" in bundleInfo:
481  command += bundleInfo["extra"].split()
482  newManifest = json.loads(runCommand(command, pipe=True))
483 
484  print(bundleInfo["name"])
485  os.chdir(bundleInfo["dir"])
486 
487  # list files to upload
488  fileList = []
489  for f, info in newManifest["files"].items():
490  if info["tag"] == options.tag:
491  fileList.append(f)
492 
493  # check if files were deleted
494  filesWereDeleted = False
495  if bundleExists(configInfo["tag"], bundleInfo["name"]):
496  command = ["scp", "-q", "%s:%s/%s/%s/%s" % (remoteHosts[0], remoteRepoDir, configInfo["tag"], bundleInfo["name"], manifestFilename), "/tmp"]
497  runCommand(command)
498  with open("/tmp/" + manifestFilename, 'rb') as manifest:
499  currentManifest = json.load(manifest)
500 
501  for f, info in currentManifest["files"].items():
502  if f not in newManifest["files"]:
503  filesWereDeleted = True
504  break
505  os.remove("/tmp/" + manifestFilename)
506 
507  if fileList or filesWereDeleted:
508  # save manifest file
509  with open(manifestFilename, 'w') as outfile:
510  json.dump(newManifest, outfile, indent=4, sort_keys=True)
511 
512  # make tmp file with list of files to upload
513  tmpFilename = "/tmp/changeList.txt"
514  with open(tmpFilename, 'w') as changeFile:
515  changeFile.write(manifestFilename)
516  changeFile.write("\n")
517  for f in fileList:
518  if not os.path.islink(f):
519  changeFile.write(f)
520  changeFile.write("\n")
521 
522  # upload files
523  for host in remoteHosts:
524  command = ["ssh", "-q", host, "mkdir -p %s/%s/%s" % (remoteRepoDir, options.tag, bundleInfo["name"]) ]
525  runCommand(command)
526 
527  command = ["rsync", "-hav", "--files-from=" + tmpFilename, ".", "%s:%s/%s/%s/" % (host, remoteRepoDir, options.tag, bundleInfo["name"]) ]
528  runCommand(command)
529  os.remove(tmpFilename)
530 
531  else:
532  print(" No files Changed, creating symbolic link")
533 
534  # add tag to manifest
535  command = [manifestCommand, "add-tag", options.tag]
536  newManifest = json.loads(runCommand(command, pipe=True))
537  with open(manifestFilename, 'w') as outfile:
538  json.dump(newManifest, outfile, indent=4, sort_keys=True)
539 
540  firstTag = newManifest['tags'][0]
541 
542  # update remote manifest and make symbolic link
543  for host in remoteHosts:
544  command = ["ssh", "-q", host, "mkdir -p %s/%s" % (remoteRepoDir, options.tag)]
545  runCommand(command)
546 
547  command = [ "scp", "-q", manifestFilename, "%s:%s/%s/%s/%s" % (host, remoteRepoDir, firstTag, bundleInfo["name"], manifestFilename) ]
548  runCommand(command)
549 
550  command = ["ssh", "-q", host, "ln -s ../%s/%s %s/%s/%s" %
551  (firstTag, bundleInfo["name"],
552  remoteRepoDir, options.tag, bundleInfo["name"]) ]
553  runCommand(command)
554 
555 def pull(options, bundleInfo):
556  print(bundleInfo["name"])
557  if bundleExists(options.tag, bundleInfo["name"]):
558  os.chdir(configInfo["baseDir"])
559  command = [manifestCommand, "download",
560  "-n", bundleInfo["name"],
561  "-t", options.tag,
562  "-d", bundleInfo["dir"]]
563  runCommand(command)
564  if options.clean:
565  command = [manifestCommand, "clean", bundleInfo["dir"]]
566  if "extra" in bundleInfo:
567  command += bundleInfo["extra"].split()
568  runCommand(command)
569  else:
570  print("Warning: Bundle %s does not exist in tag %s on the server" %(bundleInfo["name"], options.tag))
571  if options.clean:
572  if os.path.isdir(bundleInfo["dir"]):
573  print("Cleaning bundle directory", bundleInfo["dir"])
574  shutil.rmtree(bundleInfo["dir"])
575 
576 def list_tags(options):
577  command = ["ssh", "-q", remoteHosts[0], "ls " + remoteRepoDir]
578  runCommand(command)
579 
580 
581 def run():
582  global configInfo
583  global startingDir
584 
585  parser = argparse.ArgumentParser(description="Work on the minifest directories for OCSSW")
586  parser.set_defaults(func=status)
587  subparsers = parser.add_subparsers()
588 
589  add_subparser_init_dir(subparsers)
590  add_subparser_diff(subparsers)
591  add_subparser_difftool(subparsers)
592  add_subparser_status(subparsers)
593  add_subparser_push(subparsers)
594  add_subparser_pull(subparsers)
595  add_subparser_list(subparsers)
596 
597  options = parser.parse_args()
598 
599  # check if we are initializing a new OCSSW manifest
600  if options.func == init_dir:
601  options.func(options)
602  return 0
603 
604  # check if we are listing the tags
605  if options.func == list_tags:
606  options.func(options)
607  return 0
608 
609  # if making a new tag, make sure tag does not exist or the --overwite flag is used
610  if options.func == push:
611  if tagExists(options.tag):
612  if options.overwrite:
613  for host in remoteHosts:
614  command = ["ssh", "-q", host, "rm -rf " + remoteRepoDir + "/" + options.tag]
615  runCommand(command)
616  else:
617  print("Error: tag", options.tag, "exists.")
618  print("Use --overwrite (-o) to continue (will delete old version)")
619  return 1
620 
621  # find the config file
622  baseDir = os.path.abspath(os.getcwd())
623  startingDir = baseDir
624  while(baseDir != "/"):
625  fileName = os.path.join(baseDir, configInfoFilename)
626  if os.path.exists(fileName):
627  with open(fileName, 'rb') as configFile:
628  configInfo = json.load(configFile)
629  if configInfo["baseDir"] != baseDir:
630  print("Error: config file has been moved")
631  print(" real baseDir = ", baseDir)
632  print(" configFile baseDir =", configInfo["baseDir"])
633  return 1
634  else:
635  break
636  baseDir = os.path.dirname(baseDir)
637  if baseDir == "/":
638  print("Error: manifest_ocssw config file not found in current or parent directories")
639  print(" Run: manifest_ocssw init")
640  return 1
641 
642  print("current tag =", configInfo["tag"])
643 
644  #load local bundleList json file
645  localBundleList = None
646  localBundleListFile = "%s/%s" % (baseDir, BUNDLELIST_BASENAME)
647  if not os.path.isfile(localBundleListFile):
648  print("Must have a %s file in the root directory" % (BUNDLELIST_BASENAME))
649  sys.exit(1)
650  with open(localBundleListFile, 'rb') as jsonFile:
651  localBundleList = json.load(jsonFile)
652  if not localBundleList:
653  print("Must have a valid %s file in the root directory" % (BUNDLELIST_BASENAME))
654  sys.exit(1)
655 
656  # download remote bundleList json files
657  remoteBundleList = None
658  remoteBundleListFile = "/tmp/%s" % (BUNDLELIST_BASENAME)
659  remoteManifest = None
660  remoteManifestFile = "/tmp/%s" % (MANIFEST_BASENAME)
661  tag = configInfo["tag"]
662  if options.func == pull:
663  tag = options.tag
664  elif options.func == status:
665  if hasattr(options, "tag") and options.tag:
666  tag = options.tag
667 
668  if tagExists(tag):
669  # download manifest file first
670  command = ["scp", "-q", "%s:%s/%s/root/%s" %
671  (remoteHosts[0], remoteRepoDir, tag, MANIFEST_BASENAME),
672  remoteManifestFile]
673  runCommand(command)
674  with open(remoteManifestFile, 'rb') as jsonFile:
675  remoteManifest = json.load(jsonFile)
676  os.remove(remoteManifestFile)
677 
678  # download bundleList
679  command = ["scp", "-q", "%s:%s/%s/root/%s" %
680  (remoteHosts[0], remoteRepoDir, remoteManifest['files'][BUNDLELIST_BASENAME]['tag'],
681  BUNDLELIST_BASENAME), remoteBundleListFile]
682  runCommand(command)
683  with open(remoteBundleListFile, 'rb') as jsonFile:
684  remoteBundleList = json.load(jsonFile)
685  os.remove(remoteBundleListFile)
686 
687  # check for new local bundles
688  for bundleInfo in localBundleList:
689  if not findBundleInList(bundleInfo["name"], remoteBundleList):
690  print("New local bundle -", bundleInfo["name"])
691 
692  # check for deleted bundles
693  if remoteBundleList:
694  for bundleInfo in remoteBundleList:
695  if not findBundleInList(bundleInfo["name"], localBundleList):
696  print("Deleted local bundle -", bundleInfo["name"])
697 
698  bundleList = localBundleList
699  if options.func == pull:
700  bundleList = remoteBundleList
701 
702  # actually run the function
703  #for bundleInfo in bundleList:
704  for bundleInfo in bundleList:
705  options.func(options, bundleInfo)
706 
707  # update the tag in the configInfo
708  if options.func == push or options.func == pull:
709  configInfo["tag"] = options.tag
710 
711  # save as a JSON file
712  fileName = os.path.join(baseDir, configInfoFilename)
713  with open(fileName, 'w') as outfile:
714  json.dump(configInfo, outfile, indent=4, sort_keys=True)
715 
716  return 0
717 
718 
719 if __name__ == "__main__":
720  sys.exit(run())
721 
manifest_ocssw.add_subparser_status
def add_subparser_status(subparsers)
Definition: manifest_ocssw.py:131
manifest_ocssw.list_tags
def list_tags(options)
Definition: manifest_ocssw.py:576
manifest_ocssw.add_subparser_push
def add_subparser_push(subparsers)
Definition: manifest_ocssw.py:136
manifest_ocssw.run
def run()
Definition: manifest_ocssw.py:581
manifest_ocssw.init_dir
def init_dir(options)
Definition: manifest_ocssw.py:153
manifest_ocssw.diff
def diff(options, bundleInfo)
Definition: manifest_ocssw.py:216
manifest_ocssw.findBundleInList
def findBundleInList(name, bundleList)
Definition: manifest_ocssw.py:209
manifest_ocssw.push
def push(options, bundleInfo)
Definition: manifest_ocssw.py:474
manifest_ocssw.status
def status(options, bundleInfo)
Definition: manifest_ocssw.py:366
manifest_ocssw.add_subparser_pull
def add_subparser_pull(subparsers)
Definition: manifest_ocssw.py:142
manifest_ocssw.bundleExists
def bundleExists(tag, name)
Definition: manifest_ocssw.py:201
manifest_ocssw.runCommand
def runCommand(command, pipe=False, shellVal=False)
Definition: manifest_ocssw.py:180
print
void print(std::ostream &stream, const char *format)
Definition: PrintDebug.hpp:38
manifest_ocssw.add_subparser_list
def add_subparser_list(subparsers)
Definition: manifest_ocssw.py:148
manifest_ocssw.add_subparser_init_dir
def add_subparser_init_dir(subparsers)
Definition: manifest_ocssw.py:114
manifest_ocssw.add_subparser_diff
def add_subparser_diff(subparsers)
Definition: manifest_ocssw.py:119
manifest_ocssw.pull
def pull(options, bundleInfo)
Definition: manifest_ocssw.py:555
manifest_ocssw.tagExists
def tagExists(tag)
Definition: manifest_ocssw.py:192
while
while(++r<=NROOTS)
Definition: decode_rs.h:169
manifest_ocssw.add_subparser_difftool
def add_subparser_difftool(subparsers)
Definition: manifest_ocssw.py:125


ABOUT

Our Team

Contact Us

DATA

Get Started

Find Data

Use Data

TOOLS

SeaBASS

SeaDAS

Giovanni Data Visualizer

Worldview Mapping Interface

Mission Quality Monitor

HELP

How to Use Earthdata Search

Level 3 & 4 Browser Help

File Search Help

Search & Download Help

Earthdata Forum

Join Mailing List

How to Cite Data

CONNECT WITH US
COMPLIANCE

Web Privacy Policy

Data & Information Policy

Communications Policy

Accessibility

Freedom of Information Act

USA.gov

Responsible NASA Official: Sean Bailey
Curator: OceanColor Webmaster