In [18]:
%matplotlib inline
import sys,os
sys.path.append('/home1/datahome/oarcher/sarwing_cluster/post_processing/common')
sys.path.append('/home1/datahome/oarcher/sarwing_cluster/tools')
import matplotlib.pyplot as plt
import numpy as np
import numpy.ma as ma
import swFilter
import owi
import colormap_ext
import pathurl
from glob import glob
from IPython.display import display, Markdown
import warnings
warnings.filterwarnings("ignore", category=RuntimeWarning) # Nan
#float compare diff. Less is considered equal
maxDiffError=1e-9
# number of diff that must raise an alert
nDiffAlert=1
# diff Nan that must raise an alert
nNanAlert=1
leftCompare ="/home/datawork-cersat-public/cache/public/ftp/project/sarwing/processings/2a2fc9d/default/reports/nesz_flat_test"
rightCompare="/home/datawork-cersat-public/cache/public/ftp/project/sarwing/processings/4cad967/nesz_flat/reports/nesz_flat_test"
leftCompare=pathurl.toPath(leftCompare)
rightCompare=pathurl.toPath(rightCompare)
try:
with open("%s/outDir.txt" % leftCompare) as f:
leftOutDir = [line.rstrip('\n') for line in f]
except:
leftOutDir = [leftCompare]
try:
with open("%s/outDir.txt" % rightCompare) as f:
rightOutDir = [line.rstrip('\n') for line in f]
except:
rightOutDir = [rightCompare]
safes=list(set([os.path.basename(p) for p in leftOutDir + rightOutDir]))
keepSafes=[]
#keepSafes=["S1A_EW_GRDM_1SDV_20180117T143557_20180117T143701_020195_022745_93C1.SAFE","S1A_EW_GRDM_1SDV_20160826T133953_20160826T134056_012772_0141F0_B87C.SAFE"]
#keepSafes=["S1A_EW_GRDM_1SDV_20160826T133953_20160826T134056_012772_0141F0_B87C.SAFE"]
if keepSafes:
safes=[safe for safe in safes if safe in keepSafes]
print "%d SAFES to compare" % len(safes)
In [11]:
def getPath(safe,paths):
__path=None
for _path in paths:
if safe in _path:
__path=_path
break
return __path
In [ ]:
# optionnal : filter test
differSafes=[]
differKeys=[]
for safe in safes:
leftSafeDir=getPath(safe,leftOutDir)
rightSafeDir=getPath(safe,rightOutDir)
display(Markdown('# %s' % safe))
filterFiles=set([ os.path.basename(f) for f in glob("%s/*filter.nc" % leftSafeDir) + glob("%s/*filter.nc" % rightSafeDir)])
for filterFile in filterFiles:
leftFilterFile="%s/%s" % (leftSafeDir , filterFile)
rightFilterFile="%s/%s" % (rightSafeDir , filterFile)
if not os.path.exists(leftFilterFile):
print "%s does not exists. skip" % leftFilterFile
continue
if not os.path.exists(rightFilterFile):
print "%s does not exists. skip" % rightFilterFile
continue
leftFilterDS=swFilter.readFile(leftFilterFile)
rightFilterDS=swFilter.readFile(rightFilterFile)
keys=set(leftFilterDS.keys() + rightFilterDS.keys())
for key in keys:
plot=False
leftNan=np.isnan(leftFilterDS[key])
rightNan=np.isnan(rightFilterDS[key])
leftNanCount=np.count_nonzero(leftNan)
rightNanCount=np.count_nonzero(rightNan)
if abs(leftNanCount - rightNanCount) > nNanAlert:
#print "Nan count differ for key %s. left : %d , right : %d" % (key , leftNanCount , rightNanCount)
plot=True
diff=np.abs(np.nan_to_num(leftFilterDS[key]) - np.nan_to_num(rightFilterDS[key]))
common_mask=np.logical_and(ma.getmaskarray(leftNan),ma.getmaskarray(rightNan))
diff[common_mask]=0.0
ndiff=np.count_nonzero(diff > maxDiffError)
if ndiff > nDiffAlert:
maxDiff=np.nanmax(diff)
#print "found %d differences on key %s. max diff is %e" % (ndiff, key, maxDiff)
plot=True
if plot:
differSafes.append(safe)
differKeys.append(key)
fig=plt.figure(figsize=(25,15),edgecolor='blue')
#fig.suptitle('%s/%s\n%s' % (safe , filterFile , key ), fontsize=12)
display(Markdown('---'))
display(Markdown('### %s' % safe))
display(Markdown('#### %s' % (filterFile)))
display(Markdown('##### %s' % key))
display(Markdown('Nan count. Left : %d , Right : %d' % (leftNanCount , rightNanCount)))
display(Markdown('Diff count: %d (max %e)' % (ndiff , maxDiff)))
cmap=plt.get_cmap('gray')
cmap.set_bad(color='red')
#mini=np.min([np.min(leftFilterDS[key]),np.min(rightFilterDS[key])])
#maxi=np.max([np.max(leftFilterDS[key]),np.max(rightFilterDS[key])])
plt.subplot(1,3,1)
plt.title('left')
plt.pcolormesh(leftFilterDS[key],cmap=cmap)
plt.colorbar(orientation='horizontal')
#xyLeftNan= np.argwhere(leftNan)
#plt.scatter(xyLeftNan[:,0],xyLeftNan[:,1],c='r')
plt.subplot(1,3,2)
plt.title('diff')
plt.pcolormesh(diff,cmap=cmap)
plt.colorbar(orientation='horizontal')
plt.subplot(1,3,3)
plt.title('right')
plt.pcolormesh(rightFilterDS[key],cmap=cmap)
plt.colorbar(orientation='horizontal')
#xyRightNan= np.argwhere(rightNan)
#plt.scatter(xyRightNan[:,0],xyRightNan[:,1],c='r')
plt.show()
plt.close()
differSafes=list(set(differSafes))
differKeys=list(set())
display(Markdown('### Found %d/%d SAFES difference:' % (len(differSafes), len(safes) )))
for safe in differSafes:
display(Markdown('%s' % safe))
display(Markdown('### differences where found on:' ))
for key in differKeys:
display(Markdown('%s' % key))
In [19]:
# optionnal : owi test
differSafes=[]
differKeys=[]
for safe in safes:
leftSafeDir=getPath(safe,leftOutDir)
rightSafeDir=getPath(safe,rightOutDir)
#display(Markdown('# %s' % safe))
owiFiles=set([ os.path.basename(f) for f in glob("%s/*owi*.nc" % leftSafeDir) + glob("%s/*owi*.nc" % rightSafeDir)])
for owiFile in owiFiles:
leftOwiFile="%s/%s" % (leftSafeDir , owiFile)
rightOwiFile="%s/%s" % (rightSafeDir , owiFile)
if not os.path.exists(leftOwiFile):
print "%s does not exists. skip" % leftOwiFile
continue
if not os.path.exists(rightOwiFile):
print "%s does not exists. skip" % rightOwiFile
continue
leftOwiDS=owi.readFile(leftOwiFile)
rightOwiDS=owi.readFile(rightOwiFile)
keys=set(leftOwiDS.keys() + rightOwiDS.keys())
keys=['owiInversionTables_UV/owiWindSpeed_Tab_dualpol_2steps']
for key in keys:
plot=False
leftNan=np.isnan(leftOwiDS[key])
rightNan=np.isnan(rightOwiDS[key])
leftNanCount=np.count_nonzero(leftNan)
rightNanCount=np.count_nonzero(rightNan)
if abs(leftNanCount - rightNanCount) > nNanAlert:
#print "Nan count differ for key %s. left : %d , right : %d" % (key , leftNanCount , rightNanCount)
plot=True
common_mask=np.logical_and(ma.getmaskarray(leftNan),ma.getmaskarray(rightNan))
diff=np.abs(np.nan_to_num(leftOwiDS[key]) - np.nan_to_num(rightOwiDS[key]))
diff[common_mask]=0.0
ndiff=np.count_nonzero(diff > maxDiffError)
if ndiff >= nDiffAlert:
maxDiff=np.nanmax(diff)
#print "found %d differences on key %s. max diff is %e" % (ndiff, key, maxDiff)
#print np.nan_to_num(leftOwiDS[key][-1,0]),diff[-1,0],rightOwiDS[key][-1,0]
plot=True
if plot:
differSafes.append(safe)
differKeys.append(key)
fig=plt.figure(figsize=(25,15),edgecolor='blue')
#fig.suptitle('%s/%s\n%s' % (safe , filterFile , key ), fontsize=12)
display(Markdown('---'))
display(Markdown('### %s' % safe))
display(Markdown('#### %s' % (owiFile)))
display(Markdown('##### %s' % key))
display(Markdown('Nan count. Left : %d , Right : %d' % (leftNanCount , rightNanCount)))
display(Markdown('Diff count: %d (max %e)' % (ndiff , maxDiff)))
#cmap=plt.get_cmap('gray')
#cmap=plt.get_cmap('jet')
cmap = plt.get_cmap("high_wind_speed")
cmap.set_bad(color='red')
#mini=np.min([np.min(leftFilterDS[key]),np.min(rightFilterDS[key])])
#maxi=np.max([np.max(leftFilterDS[key]),np.max(rightFilterDS[key])])
vmin=None
vmax=None
if 'WindSpeed' in key:
vmin=0
vmax=80
plt.subplot(1,3,1)
plt.title('left')
plt.pcolormesh(leftOwiDS[key],vmin=vmin,vmax=vmax,cmap=cmap)
plt.colorbar(orientation='horizontal')
#xyLeftNan= np.argwhere(leftNan)
#plt.scatter(xyLeftNan[:,0],xyLeftNan[:,1],c='r')
plt.subplot(1,3,2)
#plt.title('diff')
#plt.pcolormesh(diff,cmap=cmap)
#plt.colorbar(orientation='horizontal')
plt.title('scatter diff')
plt.xlabel('left')
plt.ylabel('right')
plt.scatter(leftOwiDS[key],rightOwiDS[key])
plt.subplot(1,3,3)
plt.title('right')
plt.pcolormesh(rightOwiDS[key],vmin=vmin,vmax=vmax,cmap=cmap)
plt.colorbar(orientation='horizontal')
#xyRightNan= np.argwhere(rightNan)
#plt.scatter(xyRightNan[:,0],xyRightNan[:,1],c='r')
plt.show()
plt.close()
differSafes=list(set(differSafes))
differKeys=list(set())
display(Markdown('### Found %d/%d SAFES difference:' % (len(differSafes), len(safes) )))
for safe in differSafes:
display(Markdown('%s' % safe))
display(Markdown('### differences where found on:' ))
for key in differKeys:
display(Markdown('%s' % key))
