Merge pull request #472 from Caltech-IPAC/DM-11883-UpgradeFITSSPackage

Dm 11883 upgrade the nom.tam.fits java package in Firefly

Author: cwang2016

buildScript/depends.gincl

@@ -30,7 +30,20 @@ ext {
               ':websocket-api']
 
   mysql_lib = ':mysql-connector-java-5.1.6-bin'
-  starlink_lib = [':rayreg', ':regclient', ':stil']
+  starlink_lib = [':rayreg',
+                  ':starlink-registry-1.2+2016.05.03',
+                  ':jcdf-1.2.2',
+                  ':eag-healpix-2017.02.26',
+                  ':ndx', ':mirage', ':gbin',
+                  ':vecmath-1.5.2',
+                  ':commons-compress-1.13',
+                  ':starlink-pal-1.0.1+2016.08.11',
+                  ':starlink-cdf-1.0+2017.01.04',
+                  ':starlink-connect-0.1+2016.05.03',
+                  ':starlink-fits-0.1+2017.03.29',
+                  ':starlink-table-3.2',
+                  ':starlink-util-1.0+2017.03.17',
+                  ':starlink-votable-2.0+2016.12.22']
   xbeanfinder_lib = [':xbean-finder-4.1', ':asm-5.0.3']
   spring_lib = [':spring-beans', ':spring-core', ':spring-jdbc', ':spring-tx', ':spring-web']
   fits_io_lib = [':fits_io']

jars/fits-io/fits_io.jar

@@ -10,6 +10,7 @@
 import edu.caltech.ipac.util.DataType;
 import edu.caltech.ipac.util.StringUtils;
 import nom.tam.fits.FitsException;
+import nom.tam.fits.FitsFactory;
 import uk.ac.starlink.table.*;
 
 import java.io.File;
@@ -212,14 +213,28 @@ public static List<DataGroup> convertFitsToDataGroup(String fits_filename,
      * @return List of star tables.
      * @throws IOException
      */
-    public static List getStarTableList(String fits_filename) throws IOException {
-
-        TableSequence tseq;
-        StarTableFactory stFactory = new StarTableFactory();
-        tseq = stFactory.makeStarTables(fits_filename, null);
-        List tList = new ArrayList();
-        for (StarTable tbl; (tbl = tseq.nextTable()) != null; ) {
-            tList.add(tbl);
+    public static List getStarTableList(String fits_filename) {
+
+        ArrayList<StarTable> tList = new ArrayList<>();
+
+        // disable long string for HeaderCard create while collecting tables from StarTableFactory to work around the exception error
+        // sent from nom.tam.fits.
+        FitsFactory.setLongStringsEnabled(false);
+
+        try {
+            TableSequence tseq;
+            StarTableFactory stFactory = new StarTableFactory();
+            tseq = stFactory.makeStarTables(fits_filename, null);
+
+            for (StarTable tbl; (tbl = tseq.nextTable()) != null; ) {
+                DescribedValue dValue = tbl.getParameterByName("ZIMAGE");   // check if 'BINTABLE' is actually compressed image HDU
+                if (dValue == null || dValue.getValue() != Boolean.TRUE) {
+                    tList.add(tbl);
+                }
+            }
+
+        } catch (Exception e) {
+            logger.error("unable to get table from fits file " + fits_filename + ", reason: " + e.getMessage());
         }
         return tList;
     }

jars/starlink/commons-compress-1.13.jar

@@ -440,6 +440,7 @@ static void initState(PlotState state,
             if (state.getContextString() == null) {
                 state.setContextString(CtxControl.makeCachedCtx());
             }
+            state.setTileCompress(fi.getFitsRead().isTileCompress(), band);
             state.setOriginalImageIdx(fi.getOriginalImageIdx(), band);
             PlotStateUtil.setOriginalFitsFile(state, fi.getOriginalFile(), band);
             PlotStateUtil.setWorkingFitsFile(state, fi.getWorkingFile(), band);

jars/starlink/eag-healpix-2017.02.26.jar

@@ -379,6 +379,7 @@ public static JSONObject serializeBandState(BandState b) {
         map.put("fitsHeader", serializeClientFitsHeader(b.getHeader()));
         map.put("bandVisible", b.isBandVisible());
         map.put("multiImageFile", b.isMultiImageFile());
+        map.put("tileCompress", b.isTileCompress());
         map.put("cubeCnt", b.getCubeCnt());
         map.put("cubePlaneNumber", b.getCubePlaneNumber());
         return map;
@@ -398,6 +399,7 @@ public static BandState deserializeBandState(JSONObject map) {
             b.setFitsHeader(deserializeClientFitsHeader((JSONObject)map.get("fitsHeader")));
             b.setBandVisible((Boolean) map.get("bandVisible"));
             b.setMultiImageFile((Boolean) map.get("multiImageFile"));
+            b.setTileCompress((Boolean) map.get("tileCompress"));
             b.setCubeCnt(getInt(map,"cubeCnt"));
             b.setCubePlaneNumber(getInt(map, "cubePlaneNumber"));
             return b;

jars/starlink/gbin.jar

@@ -250,7 +250,12 @@ private static boolean isDirectFluxAccessAvailable(PlotState state) {
             if (state.getWorkingFitsFileStr(b).endsWith("gz")) {
                 return false;
             }
+
+            if (state.isTileCompress(b)) {
+                return false;
+            }
         }
+
         return true;
 
 
@@ -303,7 +308,6 @@ public static String[] getFlux(PlotState stateAry[], ImagePt ipt) {
         // 3. return all the gathered fluxes
         return fluxList.toArray(new String[fluxList.size()]);
     }
-
     public static String[] getFileFlux(FileAndHeaderInfo fileAndHeader[], ImagePt ipt) {
         try {
             String retval[] = new String[fileAndHeader.length];

jars/starlink/jcdf-1.2.2.jar

@@ -35,6 +35,7 @@ public class BandState implements Serializable, HandSerialize {
     private String fitsHeaderSerialize = null; // Serialized ClientFitsHeader
     private boolean bandVisible = true;
     private boolean multiImageFile = false;
+    private boolean tileCompress = false;
     private int     cubeCnt = 0;
     private int     cubePlaneNumber = 0;
 
@@ -62,6 +63,11 @@ public void setMultiImageFile(boolean multiImageFile) {
         this.multiImageFile = multiImageFile;
     }
 
+    public boolean isTileCompress() {return tileCompress; }
+    public void setTileCompress(boolean tCompress) {
+        this.tileCompress = tCompress;
+    }
+
     public int getCubePlaneNumber() { return cubePlaneNumber; }
     public void setCubePlaneNumber(int cubePlaneNumber) { this.cubePlaneNumber = cubePlaneNumber; }
 
@@ -165,6 +171,7 @@ public String toString() {
                 fitsHeaderSerialize,
                 bandVisible +"",
                 multiImageFile+"",
+                tileCompress+"",
                 cubeCnt+"",
                 cubePlaneNumber+"");
     }
@@ -186,6 +193,7 @@ public static BandState parse(String s) {
             ClientFitsHeader header=  ClientFitsHeader.parse(sAry[i++]);
             boolean bandVisible=    Boolean.parseBoolean(sAry[i++]);
             boolean multiImageFile= Boolean.parseBoolean(sAry[i++]);
+            boolean tileCompress = Boolean.parseBoolean(sAry[i++]);
             int cubeCnt=            Integer.parseInt(sAry[i++]);
             int cubePlaneNumber=    Integer.parseInt(sAry[i++]);
             if (req!=null && header!=null ) {
@@ -200,6 +208,7 @@ public static BandState parse(String s) {
                 retval.setFitsHeader(header);
                 retval.setBandVisible(bandVisible);
                 retval.setMultiImageFile(multiImageFile);
+                retval.setTileCompress(tileCompress);
                 retval.setCubeCnt(cubeCnt);
                 retval.setCubePlaneNumber(cubePlaneNumber);
             }

jars/starlink/mirage.jar

@@ -229,6 +229,15 @@ public boolean isMultiImageFile(Band band) {
     public boolean isMultiImageFile() { return get(firstBand()).isMultiImageFile(); }
     public void setMultiImageFile(boolean multiImageFile, Band band) { get(band).setMultiImageFile(multiImageFile); }
 
+    public boolean isTileCompress(Band band) {
+        if (band==null) band= firstBand();
+        return get(band).isTileCompress();
+    }
+
+    @JsNoExport
+    public boolean isTileCompress() { return get(firstBand()).isTileCompress(); }
+    public void setTileCompress(boolean tileCompress, Band band) { get(band).setTileCompress(tileCompress); }
+
     public int getCubeCnt(Band band) {
         if (band==null) band= firstBand();
         return get(band).getCubeCnt();
@@ -344,6 +353,10 @@ public Map<String,String> originKeyValues() {
             keyVals.put("multiImageFile", firstBandState.isMultiImageFile()+"");
             keyVals.put("imageIdx", firstBandState.getImageIdx()+"");
         }
+        if (firstBandState.isTileCompress()) {
+            keyVals.put("tileCompress", firstBandState.isTileCompress()+"");
+            keyVals.put("imageIdx", firstBandState.getImageIdx()+"");
+        }
         if (!firstBandState.isFileOriginal()) {
             keyVals.put("originalFile",firstBandState.getOriginalFitsFileStr());
             if (firstBandState.isMultiImageFile()) {

jars/starlink/ndx.jar

@@ -60,6 +60,7 @@ public interface PlotStateCI {
 
 
     boolean isMultiImageFile(Band band);
+    boolean isTileCompress(Band band);
 
     int getCubeCnt(Band band);
 

jars/starlink/starlink-cdf-1.0+2017.01.04.jar

@@ -4,6 +4,7 @@
 package edu.caltech.ipac.util;
 
 import edu.caltech.ipac.astro.IpacTableWriter;
+import nom.tam.image.compression.hdu.CompressedImageHDU;
 import org.json.simple.JSONObject;
 import nom.tam.fits.Fits;
 import nom.tam.fits.FitsException;
@@ -123,7 +124,8 @@ public static DataGroup fitsHeaderToDataGroup(String fitsFile) {
                 JSONObject extensionInfo;
                 List<List<String>> headerRows = new ArrayList<>();
 
-                String name = "NoName";
+                Boolean isCompressed = (hdu instanceof CompressedImageHDU);
+                String name = isCompressed ? "CompressedImage" : "NoName";
                 String type = "IMAGE";
 
                 Header hduHeader = hdu.getHeader();
@@ -144,7 +146,7 @@ public static DataGroup fitsHeaderToDataGroup(String fitsFile) {
                             type = "";
                         }
                     } else {
-                        if (key.equals("XTENSION")) {
+                        if (key.equals("XTENSION") && (!isCompressed)) {
                             type = val;
                         } else if (key.equals("NAME") || key.equals("EXTNAME")) {
                             name = val;
@@ -169,6 +171,7 @@ public static DataGroup fitsHeaderToDataGroup(String fitsFile) {
 
                     name = String.format("%s/%d cols x %d rows", name, colNo, rowNo);
                 }
+
                 row.setDataElement(cols.get(1), (index == 0) ? "Primary" : name);
                 row.setDataElement(cols.get(2), type);
                 dg.add(row);

jars/starlink/starlink-connect-0.1+2016.05.03.jar

@@ -13,6 +13,7 @@
 import nom.tam.fits.ImageHDU;
 import nom.tam.util.ArrayFuncs;
 import nom.tam.util.Cursor;
+import nom.tam.image.compression.hdu.*;
 
 import java.io.DataOutputStream;
 import java.io.FileOutputStream;
@@ -21,6 +22,7 @@
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.nio.Buffer;
 
 
 /**
@@ -75,6 +77,7 @@ public class FitsRead implements Serializable {
     private int indexInFile = -1;  // -1 unknown, >=0 index in file
     private Histogram hist;
     private  double defBetaValue= Double.NaN;
+    private boolean tileCompress = false;
 
     private static ArrayList<Integer> SUPPORTED_BIT_PIXS = new ArrayList<Integer>(Arrays.asList(8, 16, 32, -32, -64));
 
@@ -155,6 +158,7 @@ public static FitsRead[] createFitsReadArray(Fits fits, boolean clearHdu)
             throw new FitsException("Bad format in FITS file");
         }
 
+        boolean isCompressed = anyCompressedImage(HDUs);
 
         ArrayList<BasicHDU> HDUList = getHDUList(HDUs);
 
@@ -165,6 +169,7 @@ public static FitsRead[] createFitsReadArray(Fits fits, boolean clearHdu)
         for (int i = 0; i < HDUList.size(); i++) {
             fitsReadAry[i] = new FitsRead((ImageHDU) HDUList.get(i), clearHdu);
             fitsReadAry[i].indexInFile = i;
+            fitsReadAry[i].tileCompress = isCompressed;
         }
 
         return fitsReadAry;
@@ -188,6 +193,8 @@ public static FitsRead[] createFitsReadArray(Fits fits, BasicHDU hdu)
         }
 
         BasicHDU[] HDUs={hdu};
+
+        boolean isCompressed = anyCompressedImage(HDUs);
         ArrayList<BasicHDU> HDUList = getHDUList(HDUs);
 
         if (HDUList.size() == 0)
@@ -197,7 +204,7 @@ public static FitsRead[] createFitsReadArray(Fits fits, BasicHDU hdu)
         for (int i = 0; i < HDUList.size(); i++) {
             fitsReadAry[i] = new FitsRead( (ImageHDU) HDUList.get(i), false);
             fitsReadAry[i].indexInFile = i;
-
+            fitsReadAry[i].tileCompress = isCompressed;
         }
 
 
@@ -537,40 +544,55 @@ private static void updateHeader(Header header, int pos, long hduOffset)
     }
 
 
+    private static boolean anyCompressedImage(BasicHDU[] HDUs) {
+        for (int j = 0; j < HDUs.length; j++) {
+            if (HDUs[j] instanceof CompressedImageHDU)
+                return true;
+        }
+        return false;
+    }
+
     private static ArrayList<BasicHDU> getHDUList(BasicHDU[] HDUs) throws FitsException {
         ArrayList<BasicHDU> HDUList = new ArrayList<BasicHDU>();
 
         boolean hasExtension = HDUs.length > 1 ? true : false;
         for (int j = 0; j < HDUs.length; j++) {
-            if (!(HDUs[j] instanceof ImageHDU)) {
-                continue;   //ignor non-image extensions
+            if (!(HDUs[j] instanceof ImageHDU) && !(HDUs[j] instanceof CompressedImageHDU)) {
+                continue;   //ignore non-image extensions
             }
-            //process image HDU
-            Header header = HDUs[j].getHeader();
-            if (header == null)
-                throw new FitsException("Missing header in FITS file");
+            //process image HDU or compressed image HDU as ImageHDU
+
+            ImageHDU hdu = null;
 
+            try {
+                hdu = (HDUs[j] instanceof ImageHDU) ? (ImageHDU) HDUs[j] : ((CompressedImageHDU) HDUs[j]).asImageHDU();
+            } catch(Exception e) {
+                throw new FitsException(e.toString());
+            }
 
+            Header header = (hdu != null) ? hdu.getHeader() : null;
+            if (header == null)
+                throw new FitsException("Missing header in FITS file");
 
             int naxis = header.getIntValue("NAXIS", -1);
             boolean goodImage = isImageGood(header);
 
             if (goodImage) {
                 if (hasExtension) { // update this hdu by adding keywords/values
-                    updateHeader(header, j, HDUs[j].getFileOffset());
+                    updateHeader(header, j, hdu.getFileOffset());
                 }
 
                 int naxis3 = header.getIntValue("NAXIS3", -1);
                 if ((naxis > 2) && (naxis3 > 1)) { //it is a cube data
                     if (SUTDebug.isDebug())
                         System.out.println("GOT A FITS CUBE");
-                    BasicHDU[] splitHDUs = splitFitsCube( (ImageHDU) HDUs[j]);
+                    BasicHDU[] splitHDUs = splitFitsCube( hdu );
                     /* for each plane of cube */
                     for (int jj = 0; jj < splitHDUs.length; jj++) {
                         HDUList.add(splitHDUs[jj]);
                     }
                 } else {
-                    HDUList.add(HDUs[j]);
+                    HDUList.add(hdu);
                 }
             }
 
@@ -1673,6 +1695,7 @@ public int getExtensionNumber() {
         return extension_number;
     }
 
+    public boolean isTileCompress() { return tileCompress; }
     /**
      * The Bscale  keyword shall be used, along with the BZERO keyword, when the array pixel values are not the true  physical  values,
      * to transform the primary data array  values to the true physical values they represent, using Eq. 5.3. The value field shall contain a

jars/starlink/starlink-fits-0.1+2017.03.29.jar

@@ -58,7 +58,7 @@ public static void main(String[] args)
 	{
 	    HDU_offset =  myHDUs[i].getFileOffset();
 	    System.out.println("getFileOffset = " + myHDUs[i].getFileOffset());
-            myHDUs[i].info();
+            myHDUs[i].info(System.out);
 	    Header header = myHDUs[i].getHeader();
 	    header_size = header.getOriginalSize();
 	    System.out.println("header.getSize() = " + header.getSize());