Various bugfixes, comment changes for Krzysztof

python/lsst/ap/association/diaPipe.py

@@ -70,9 +70,9 @@ class DiaPipelineConnections(
  solarSystemObjectTable = connTypes.Input(
  doc="Catalog of SolarSolarSystem objects expected to be observable in "
  "this detectorVisit.",
- name="visitSsObjects",
+ name="expected_ssObjects",
  storageClass="DataFrame",
- dimensions=("instrument", "visit"),
+ dimensions=("instrument", "group", "detector"),
  )
  diffIm = connTypes.Input(
  doc="Difference image on which the DiaSources were detected.",

python/lsst/ap/association/mpSkyEphemerisQuery.py

@@ -22,9 +22,7 @@
 """Solar System Object Query to MPSky in place of a internal Rubin solar
 system object caching/retrieval code.
 
-Will compute the location for of known SSObjects within a visit. This code
-blocks on web requests, so should not be used as part of any real-time or
-time-sensitive system. Use in a larger pipeline at your own risk.
+Will compute the location for of known SSObjects within a visit
 """
 
 __all__ = ["MPSkyEphemerisQueryConfig", "MPSkyEphemerisQueryTask"]
@@ -36,32 +34,27 @@
 import requests
 import sys
 
-import lsst.pipe.base as pipeBase
 import lsst.pex.config as pexConfig
 from lsst.utils.timer import timeMethod
 from lsst.geom import SpherePoint
 
-from lsst.pipe.base import PipelineTask, PipelineTaskConfig, PipelineTaskConnections
-import lsst.pipe.base.connectionTypes as connTypes
-
-# Enforce an error for unsafe column/array value setting in pandas.
-pd.options.mode.chained_assignment = 'raise'
-
+from lsst.pipe.base import PipelineTask, PipelineTaskConfig, \
+ PipelineTaskConnections, connectionTypes, Struct
 
 class MPSkyEphemerisQueryConnections(PipelineTaskConnections,
  dimensions=("instrument",
  "group", "detector")):
 
- predictedRegionTime = pipeBase.connectionTypes.Input(
+ predictedRegionTime = connectionTypes.Input(
  doc="The predicted exposure region and time",
  name="regionTimeInfo",
  storageClass="RegionTimeInfo",
  dimensions={"instrument", "group", "detector"},
  )
 
- ssObjects = connTypes.Output(
+ ssObjects = connectionTypes.Output(
  doc="MPSky-provided Solar System objects observable in this detector-visit",
- name="preloaded_ssObjects",
+ name="expected_ssObjects",
  storageClass="DataFrame",
  dimensions=("instrument", "group", "detector"),
  )
@@ -82,7 +75,7 @@ class MPSkyEphemerisQueryConfig(
  "to the radius of Rubin Obs FoV in degrees",
  default=1.75
  )
- MPSkyURL = pexConfig.Field(
+ mpSkyURL = pexConfig.Field(
  dtype=str,
  doc="URL to query mpsky service",
  default="http://sdfrome001.sdf.slac.stanford.edu:3666"
@@ -96,12 +89,10 @@ class MPSkyEphemerisQueryTask(PipelineTask):
  ConfigClass = MPSkyEphemerisQueryConfig
  _DefaultName = "MPSkyEphemerisQuery"
 
- def runQuantum(self, butlerQC, inputRefs, outputRefs):
- inputs = butlerQC.get(inputRefs)
-
- outputs = self.run(**inputs)
-
- butlerQC.put(outputs, outputRefs)
+ #def runQuantum(self, butlerQC, inputRefs, outputRefs):
+ #inputs = butlerQC.get(inputRefs)
+ #outputs = self.run(**inputs)
+ #butlerQC.put(outputs, outputRefs)
 
  @timeMethod
  def run(self, predictedRegionTime):
@@ -110,8 +101,7 @@ def run(self, predictedRegionTime):
 
  Parameters
  ----------
- predictedRegionTime : `pipe.base.utils.RegionTimeInfo`
- RegionTime of the predicted exposure
+ Predicted footprint and timespan of the exposure
 
  Returns
  -------
@@ -121,50 +111,47 @@ def run(self, predictedRegionTime):
  - ``ssObjects``: `pandas.DataFrame`
  DataFrame containing Solar System Objects in field of view as
  retrieved by MPSky. The columns are as follows:
+
  ``Name``
  object name (`str`)
  ``ra``
  RA in decimal degrees (`float`)
  ``dec``
  DEC in decimal degrees (`float`)
  ``obj_poly``
- DO NOT USE until t_min issue is resolved
+ DO NOT USE until MPSky defines polynomial t_min, t_max
  ``obs_poly``
- DO NOT USE until t_min issue is resolved
+ DO NOT USE until MPSky defines polynomial t_min, t_max
  """
  # Grab the visitInfo from the raw to get the information needed on the
  # full visit.
-
- # Midpoint time of the exposure in JD
  region = predictedRegionTime.region
  timespan = predictedRegionTime.timespan
  expCenter = SpherePoint(region.getBoundingCircle().getCenter())
-
- # Make sure date is non-NaN.
  expMidPointEPOCH = (timespan.begin.mjd + timespan.end.mjd)/2
 
  # MPSky service query
- MPSkySsObjects = self._MPSkyConeSearch(expCenter, expMidPointEPOCH, self.config.queryRadiusDegrees)
+ mpSkySsObjects = self._mpSkyConeSearch(expCenter, expMidPointEPOCH, self.config.queryRadiusDegrees)
 
- return pipeBase.Struct(
- ssObjects=MPSkySsObjects,
+ return Struct(
+ ssObjects=mpSkySsObjects,
  )
 
- def _MPSkyConeSearch(self, expCenter, epochMJD, queryRadius):
- """Query MPSky ephemeris service using the exposure boresight.
+ def _mpSkyConeSearch(self, expCenter, epochMJD, queryRadius):
+ """Query MPSky ephemeris service for objects near the expected detector position
 
  Parameters
  ----------
  expCenter : `lsst.geom.SpherePoint`
- Center of Exposure RADEC [deg]
- epochJD : `float`
- Mid point JD of exposure, in UTC [EPOCH].
+ Center of search cone
+ epochMJD : `float`
+ Epoch of cone search, (MJD in UTC).
  queryRadius : `float`
  Radius of the cone search in degrees.
 
  Returns
  -------
- MPSkySsObjects : `pandas.DataFrame`
+ mpSkySsObjects : `pandas.DataFrame`
  DataFrame with Solar System Object information and RA/DEC position
  within the visit.
  """
@@ -180,39 +167,38 @@ def _MPSkyConeSearch(self, expCenter, epochMJD, queryRadius):
  }
 
  try:
- response = requests.get(self.config.MPSkyURL, params=params)
+ response = requests.get(self.config.mpSkyURL, params=params)
  response.raise_for_status()
  with pa.input_stream(memoryview(response.content)) as fp:
  fp.seek(0)
  p = pa.ipc.read_tensor(fp)
  op = pa.ipc.read_tensor(fp)
- op, p = op, p
  with pa.ipc.open_stream(fp) as reader:
  r = next(reader)
 
  ObjID = r["name"].to_numpy(zero_copy_only=False)
  ra = r["ra"].to_numpy()
  dec = r["dec"].to_numpy()
 
- MPSkySsObjects = pd.DataFrame()
- MPSkySsObjects['ObjID'] = ObjID
- MPSkySsObjects['ra'] = ra
- MPSkySsObjects['dec'] = dec
- MPSkySsObjects['obj_poly'] = list(np.zeros((len(MPSkySsObjects), 5))) # fix, eventually
- MPSkySsObjects['obs_poly'] = list(np.zeros((len(MPSkySsObjects), 5))) # fix, eventually
- MPSkySsObjects['Err(arcsec)'] = 2
- MPSkySsObjects['ssObjectId'] = [abs(hash(v)) for v in MPSkySsObjects['ObjID'].values]
- nFound = len(MPSkySsObjects)
+ mpSkySsObjects = pd.DataFrame()
+ mpSkySsObjects['ObjID'] = ObjID
+ mpSkySsObjects['ra'] = ra
+ # TODO: accept MPSky polynomials when they are well-formed.
+ mpSkySsObjects['obs_poly'] = list(np.zeros((len(mpSkySsObjects), 5)))
+ mpSkySsObjects['obj_poly'] = list(np.zeros((len(mpSkySsObjects), 5)))
+ mpSkySsObjects['dec'] = dec
+ mpSkySsObjects['Err(arcsec)'] = 2
+ mpSkySsObjects['ssObjectId'] = [abs(hash(v)) for v in mpSkySsObjects['ObjID'].values]
+ nFound = len(mpSkySsObjects)
 
  if nFound == 0:
  self.log.info("No Solar System objects found for visit.")
-
- self.log.info("%d Solar System Objects in visit", nFound)
+ else:
+  self.log.info("%d Solar System Objects in visit", nFound)
  except requests.exceptions.ConnectionError as e:
- print("failed to connect to the remote ephemerides service. details:", file=sys.stderr)
- print(e, file=sys.stderr)
- MPSkySsObjects = pd.DataFrame(
+ self.log.exception("Failed to connect to the remote ephemerides service.")
+ mpSkySsObjects = pd.DataFrame(
  columns=['ObjID', 'ra', 'dec', 'obj_poly', 'obs_poly',
  'Err(arcsec)', 'ssObjectId'])
 
- return MPSkySsObjects
+ return mpSkySsObjects

python/lsst/ap/association/ssoAssociation.py

@@ -134,7 +134,7 @@ def run(self, diaSourceCatalog, solarSystemObjects, exposure):
  diaSourceCatalog.loc[diaSourceCatalog.index[idx[0]], "ssObjectId"] = ssObject["ssObjectId"]
 
  self.log.info("Successfully associated %d SolarSystemObjects.", nFound)
- assocMask = (diaSourceCatalog["ssObjectId"] != 0) & (np.isfinite(diaSourceCatalog["ssObjectId"]))
+ assocMask = diaSourceCatalog["ssObjectId"] != 0
  return pipeBase.Struct(
  ssoAssocDiaSources=diaSourceCatalog[assocMask].reset_index(drop=True),
  unAssocDiaSources=diaSourceCatalog[~assocMask].reset_index(drop=True),