Skip to content

Fix decoding issue of git version for py3 #222

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mark-petersen merged 3 commits into from
May 9, 2019
Merged

Fix decoding issue of git version for py3 #222

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5508484
Fix decoding issue of git version for py3
xylar Apr 29, 2019
2feb378
Update utility_scripts to py3
xylar Apr 29, 2019
5681879
Update ConfigParser in COMPASS setup
xylar May 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion testing_and_setup/compass/manage_regression_suite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,7 @@ def summarize_suite(suite_tag): # {{{
os.chdir(os.path.dirname(os.path.realpath(__file__)))
git_version = subprocess.check_output(
['git', 'describe', '--tags', '--dirty'])
git_version = git_version.strip('\n')
git_version = git_version.decode('utf-8').strip('\n')
os.chdir(old_dir)
calling_command = ""
for arg in sys.argv:
Expand Down
7 changes: 5 additions & 2 deletions testing_and_setup/compass/setup_testcase.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import argparse
import xml.etree.ElementTree as ET
import subprocess
from six.moves.configparser import SafeConfigParser
from six.moves import configparser
import textwrap
import netCDF4

Expand Down Expand Up @@ -1615,7 +1615,10 @@ def get_case_name(config_file): # {{{
case_list = list()
case_list.append(0)

config = SafeConfigParser()
if sys.version_info >= (3, 2):
config = configparser.ConfigParser()
else:
config = configparser.SafeConfigParser()
config.read(args.config_file)

if not args.no_download:
Expand Down
10 changes: 6 additions & 4 deletions testing_and_setup/compass/utility_scripts/check_progress.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
checks that the current date does not exceed the end
date specified by the -e flag with format
YYYY-MM-DD_hh:mm:ss. (Note: only years between 0001
and 9999 are supported.) If this date is exceeded,
exits 1, indicating that the run should stop.
and 9999 are supported.) If this date is exceeded,
exits 1, indicating that the run should stop.
Otherwise, exits 0, indicating the run has not yet
completed.

Expand Down Expand Up @@ -52,10 +52,12 @@

A number of jobs could be chained with this script. If one of
these jobs exits with exit status 1, this could be made to
cancel dependent jobs (depending on the capabilities of the
cancel dependent jobs (depending on the capabilities of the
job scheduler). Even if not, the next job would immediately
exit once it runs, thus using negligible computing time.
"""
from __future__ import absolute_import, division, print_function, \
unicode_literals

import os
import argparse
Expand Down Expand Up @@ -91,6 +93,6 @@
inFile.close()
endDate = datetime.strptime(args.endDate, '%Y-%m-%d_%H:%M:%S')
if restartDate >= endDate:
print 'Run has completed.'
print('Run has completed.')
sys.exit(1)
sys.exit(0)
213 changes: 108 additions & 105 deletions testing_and_setup/compass/utility_scripts/compare_fields.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
#!/usr/bin/env python
import sys, os, glob, shutil
from __future__ import absolute_import, division, print_function, \
unicode_literals

import sys
import os
import numpy as np

from netCDF4 import *
from netCDF4 import Dataset as NetCDFFile
import argparse

Expand All @@ -18,153 +21,153 @@
args = parser.parse_args()

if not args.filename1:
parser.error("Two filenames are required inputs.")
parser.error("Two filenames are required inputs.")

@xylar xylar Apr 29, 2019

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lots of tabs have been replaced with spaces. It's a bad practice to have a mix of tabs and spaces in a python file, and generally tabs should be avoided.

@mark-petersen mark-petersen May 9, 2019

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks. I appreciate that.


if not args.filename2:
parser.error("Two filenames are required inputs.")
parser.error("Two filenames are required inputs.")

if not args.variable:
parser.error("Variable is a required input.")
parser.error("Variable is a required input.")

if (not args.l2_norm) and (not args.l1_norm) and (not args.linf_norm):
print "WARNING: Script will pass since no norm values have been defined."
print("WARNING: Script will pass since no norm values have been defined.")

files_exist = True

if not os.path.exists(args.filename1):
print "ERROR: File %s does not exist. Comparison will FAIL."%(args.filename1)
files_exist = False
print("ERROR: File %s does not exist. Comparison will FAIL."%(args.filename1))
files_exist = False

if not os.path.exists(args.filename2):
print "ERROR: File %s does not exist. Comparison will FAIL."%(args.filename2)
files_exist = False
print("ERROR: File %s does not exist. Comparison will FAIL."%(args.filename2))
files_exist = False

if not files_exist:
sys.exit(1)
sys.exit(1)

f1 = NetCDFFile(args.filename1,'r')
f2 = NetCDFFile(args.filename2,'r')

try:
time_length = f1.variables['xtime'].shape[0]
time_length = f1.variables['xtime'].shape[0]
except:
time_length = 1
time_length = 1

try:
field1 = f1.variables[args.variable]
field2 = f2.variables[args.variable]
field1 = f1.variables[args.variable]
field2 = f2.variables[args.variable]
except:
print "ERROR: Field '%s' does not exist in both"%(args.variable)
print " file1: %s"%(args.filename1)
print " and file2: %s"%(args.filename2)
print "Exiting with a failed comparision, since no comparision can be done but a comparison was requested."
sys.exit(1)
print("ERROR: Field '%s' does not exist in both"%(args.variable))
print(" file1: %s"%(args.filename1))
print(" and file2: %s"%(args.filename2))
print("Exiting with a failed comparision, since no comparision can be done but a comparison was requested.")
sys.exit(1)

if not field1.shape == field2.shape:
print "ERROR: Field sizes don't match in different files."
sys.exit(1)
print("ERROR: Field sizes don't match in different files.")
sys.exit(1)

linf_norm = -(sys.float_info.max)

pass_val = True

print "Beginning variable comparisons for all time levels of field '%s'. Note any time levels reported are 0-based."%(args.variable)
print("Beginning variable comparisons for all time levels of field '%s'. Note any time levels reported are 0-based."%(args.variable))
if ( args.l2_norm or args.l1_norm or args.linf_norm ):
print " Pass thresholds are:"
if ( args.l1_norm ):
print " L1: %16.14e"%(float(args.l1_norm))
if ( args.l2_norm ):
print " L2: %16.14e"%(float(args.l2_norm))
if ( args.linf_norm ):
print " L_Infinity: %16.14e"%(float(args.linf_norm))
print(" Pass thresholds are:")
if ( args.l1_norm ):
print(" L1: %16.14e"%(float(args.l1_norm)))
if ( args.l2_norm ):
print(" L2: %16.14e"%(float(args.l2_norm)))
if ( args.linf_norm ):
print(" L_Infinity: %16.14e"%(float(args.linf_norm)))

field_dims = field1.dimensions

if "Time" in field_dims:
for t in range( 0, time_length):
pass_time = True
if len(field_dims) >= 2:
diff = np.absolute(field1[t][:] - field2[t][:])
else:
diff = np.absolute(field1[t] - field2[t])

l2_norm = np.sum(diff * diff)
l2_norm = np.sqrt(l2_norm)

l1_norm = np.sum(diff)
if len(field_dims) >= 2:
l1_norm = l1_norm / np.sum(field1[t][:].shape)
l1_norm = np.max(l1_norm)

if np.amax(diff) > linf_norm:
linf_norm = np.amax(diff)

diff_str = '%d: '%(t)
if args.l1_norm:
if float(args.l1_norm) < l1_norm:
pass_time = False
diff_str = '%s l1: %16.14e '%(diff_str, l1_norm)

if args.l2_norm:
if float(args.l2_norm) < l2_norm:
pass_time = False
diff_str = '%s l2: %16.14e '%(diff_str, l2_norm)

if args.linf_norm:
if float(args.linf_norm) < linf_norm:
pass_time = False
diff_str = '%s linf: %16.14e '%(diff_str, linf_norm)

if not args.quiet:
print diff_str
elif not pass_time:
print diff_str

if not pass_time:
pass_val = False

del diff
for t in range( 0, time_length):
pass_time = True
if len(field_dims) >= 2:
diff = np.absolute(field1[t][:] - field2[t][:])
else:
diff = np.absolute(field1[t] - field2[t])

l2_norm = np.sum(diff * diff)
l2_norm = np.sqrt(l2_norm)

l1_norm = np.sum(diff)
if len(field_dims) >= 2:
l1_norm = l1_norm / np.sum(field1[t][:].shape)
l1_norm = np.max(l1_norm)

if np.amax(diff) > linf_norm:
linf_norm = np.amax(diff)

diff_str = '%d: '%(t)
if args.l1_norm:
if float(args.l1_norm) < l1_norm:
pass_time = False
diff_str = '%s l1: %16.14e '%(diff_str, l1_norm)

if args.l2_norm:
if float(args.l2_norm) < l2_norm:
pass_time = False
diff_str = '%s l2: %16.14e '%(diff_str, l2_norm)

if args.linf_norm:
if float(args.linf_norm) < linf_norm:
pass_time = False
diff_str = '%s linf: %16.14e '%(diff_str, linf_norm)

if not args.quiet:
print(diff_str)
elif not pass_time:
print(diff_str)

if not pass_time:
pass_val = False

del diff
else:
if len(field_dims) >= 2:
diff = np.absolute(field1[:] - field2[:])
else:
diff = np.absolute(field1[0] - field2[0])
if len(field_dims) >= 2:
diff = np.absolute(field1[:] - field2[:])
else:
diff = np.absolute(field1[0] - field2[0])

l2_norm = np.sum(diff * diff)
l2_norm = np.sqrt(l2_norm)
l2_norm = np.sum(diff * diff)
l2_norm = np.sqrt(l2_norm)

l1_norm = np.sum(diff)
if len(field_dims) >= 2:
l1_norm = l1_norm / np.sum(field1[:].shape)
l1_norm = np.max(l1_norm)
l1_norm = np.sum(diff)
if len(field_dims) >= 2:
l1_norm = l1_norm / np.sum(field1[:].shape)
l1_norm = np.max(l1_norm)

if np.amax(diff) > linf_norm:
linf_norm = np.amax(diff)
if np.amax(diff) > linf_norm:
linf_norm = np.amax(diff)

diff_str = ''
if args.l1_norm:
if float(args.l1_norm) < l1_norm:
pass_val = False
diff_str = '%s l1: %16.14e '%(diff_str, l1_norm)
diff_str = ''
if args.l1_norm:
if float(args.l1_norm) < l1_norm:
pass_val = False
diff_str = '%s l1: %16.14e '%(diff_str, l1_norm)

if args.l2_norm:
if float(args.l2_norm) < l2_norm:
pass_val = False
diff_str = '%s l2: %16.14e '%(diff_str, l2_norm)
if args.l2_norm:
if float(args.l2_norm) < l2_norm:
pass_val = False
diff_str = '%s l2: %16.14e '%(diff_str, l2_norm)

if args.linf_norm:
if float(args.linf_norm) < linf_norm:
pass_val = False
diff_str = '%s linf: %16.14e '%(diff_str, linf_norm)
if args.linf_norm:
if float(args.linf_norm) < linf_norm:
pass_val = False
diff_str = '%s linf: %16.14e '%(diff_str, linf_norm)

if not args.quiet:
print diff_str
elif not pass_val:
print diff_str
if not args.quiet:
print(diff_str)
elif not pass_val:
print(diff_str)

del diff
del diff

if pass_val:
sys.exit(0)
sys.exit(0)
else:
sys.exit(1)
sys.exit(1)
Loading