Merge pull request #170 from lsst/tickets/DM-37196

DM-37196: Update fields to reflect new APDB schema

data/DiaSource.yaml

@@ -7,9 +7,9 @@ funcs:
  ccdVisitId:
  functor: Column
  args: ccdVisitId
- filterName:
+ band:
  functor: Column
- args: filterName
+ args: band
  diaObjectId:
  functor: Column
  args: diaObjectId
@@ -19,22 +19,26 @@ funcs:
  parentDiaSourceId:
  functor: Column
  args: parent
- midPointTai:
+ midpointMjdTai:
  functor: Column
- args: midPointTai
+ args: midpointMjdTai
  bboxSize:
  functor: Column
  args: bboxSize
+ # TODO: is this one really necessary here?
+ time_processed: # filled in automatically
+ functor: Column
+ args: time_processed
  flags:
  functor: Column
  args: flags
  ra:
  functor: RAColumn
  # raErr: not available yet DM-15180
- decl:
+ dec:
  functor: DecColumn
- # declErr: not available yet DM-15180
- # ra_decl_Cov: not available yet
+ # decErr: not available yet DM-15180
+ # ra_dec_Cov: not available yet
  x:
  functor: Column
  args: slot_Centroid_x
@@ -65,27 +69,27 @@ funcs:
  snr:
  functor: Column
  args: snr
- psFlux:
+ psfFlux:
  functor: LocalNanojansky
  args:
  - slot_PsfFlux_instFlux
  - slot_PsfFlux_instFluxErr
  - base_LocalPhotoCalib
  - base_LocalPhotoCalibErr
- psFluxErr:
+ psfFluxErr:
  functor: LocalNanojanskyErr
  args:
  - slot_PsfFlux_instFlux
  - slot_PsfFlux_instFluxErr
  - base_LocalPhotoCalib
  - base_LocalPhotoCalibErr
- # ps_ra_cov not implemented
- # ps_dec_cov not implemented
- # psLnl not implemented
- psChi2:
+ # psfFlux_psfRa_cov not implemented
+ # psfFlux_psfDec_cov not implemented
+ # psfLnl not implemented
+ psfChi2:
  functor: Column
  args: slot_PsfFlux_chi2
- psNdata:
+ psfNdata:
  functor: Column
  args: slot_PsfFlux_npixels
  trailFlux:
@@ -98,7 +102,7 @@ funcs:
  trailRa:
  functor: Column
  args: ext_trailedSources_Naive_ra
- trailDecl:
+ trailDec:
  functor: Column
  args: ext_trailedSources_Naive_dec
  trailLength:
@@ -116,7 +120,7 @@ funcs:
  # trailLnL not implemented
  # trailChi2 not implemented
  # trailNdata not implemented
- dipMeanFlux:
+ dipoleMeanFlux:
  functor: LocalDipoleMeanFlux
  args:
  - ip_diffim_NaiveDipoleFlux_pos_instFlux
@@ -125,7 +129,7 @@ funcs:
  - ip_diffim_NaiveDipoleFlux_neg_instFluxErr
  - base_LocalPhotoCalib
  - base_LocalPhotoCalibErr
- dipMeanFluxErr:
+ dipoleMeanFluxErr:
  functor: LocalDipoleMeanFluxErr
  args:
  - ip_diffim_NaiveDipoleFlux_pos_instFlux
@@ -134,7 +138,7 @@ funcs:
  - ip_diffim_NaiveDipoleFlux_neg_instFluxErr
  - base_LocalPhotoCalib
  - base_LocalPhotoCalibErr
- dipFluxDiff:
+ dipoleFluxDiff:
  functor: LocalDipoleDiffFlux
  args:
  - ip_diffim_NaiveDipoleFlux_pos_instFlux
@@ -143,7 +147,7 @@ funcs:
  - ip_diffim_NaiveDipoleFlux_neg_instFluxErr
  - base_LocalPhotoCalib
  - base_LocalPhotoCalibErr
- dipFluxDiffErr:
+ dipoleFluxDiffErr:
  functor: LocalDipoleDiffFluxErr
  args:
  - ip_diffim_NaiveDipoleFlux_pos_instFlux
@@ -152,46 +156,46 @@ funcs:
  - ip_diffim_NaiveDipoleFlux_neg_instFluxErr
  - base_LocalPhotoCalib
  - base_LocalPhotoCalibErr
- # dipRa not implemented
- # dipDec not implemented
+ # dipoleRa not implemented
+ # dipoleDec not implemented
  # (this may be redundant with RA/DEC as the default centroid is the
  # dip model, defaulting to SdssCentroid on Dip-Fit failure.)
- dipLength:
+ dipoleLength:
  functor: ConvertPixelToArcseconds
  args:
  - ip_diffim_DipoleFit_separation
  - base_LocalWcs_CDMatrix_1_1
  - base_LocalWcs_CDMatrix_1_2
  - base_LocalWcs_CDMatrix_2_1
  - base_LocalWcs_CDMatrix_2_2
- dipAngle:
+ dipoleAngle:
  functor: Column
  args: ip_diffim_DipoleFit_orientation
- # dipCov not implemented
- # dipLnl not implemented
- dipChi2:
+ # dipoleCov not implemented
+ # dipoleLnl not implemented
+ dipoleChi2:
  functor: Column
  args: ip_diffim_DipoleFit_chi2dof
  isDipole:
  functor: Column
  args: ip_diffim_DipoleFit_flag_classification
- # dipNdata not implemented
- totFlux:
+ # dipoleNdata not implemented
+ scienceFlux:
  functor: LocalNanojansky
  args:
  - ip_diffim_forced_PsfFlux_instFlux
  - ip_diffim_forced_PsfFlux_instFluxErr
  - base_LocalPhotoCalib
  - base_LocalPhotoCalibErr
- totFluxErr:
+ scienceFluxErr:
  functor: LocalNanojanskyErr
  args:
  - ip_diffim_forced_PsfFlux_instFlux
  - ip_diffim_forced_PsfFlux_instFluxErr
  - base_LocalPhotoCalib
  - base_LocalPhotoCalibErr
- # diffFlux not implemented and likely dropped due to no snaps.
- # diffFluxErr not implemented and likely dropped due to no snaps.
+ # snapDiffFlux not implemented and likely dropped due to no snaps.
+ # snapDiffFluxErr not implemented and likely dropped due to no snaps.
  # fpBkgd not measured yet. DM-
  # fpBkgdErr not measured yet. DM-
 
@@ -247,6 +251,6 @@ funcs:
  - base_LocalWcs_CDMatrix_2_1
  - base_LocalWcs_CDMatrix_2_2
  # extendedness not implemented
- spuriousness:
+ reliability:
  functor: Column
- args: spuriousness
+ args: reliability

data/apdb-ap-pipe-afw-map.yaml

@@ -6,10 +6,10 @@
 DiaObject:
  diaObjectId: id
  ra: coord_ra
- decl: coord_dec
+ dec: coord_dec
 
 DiaSource:
  diaSourceId: id
  parentDiaSourceId: parent
  ra: coord_ra
- decl: coord_dec
+ dec: coord_dec

python/lsst/ap/association/association.py

@@ -127,7 +127,7 @@ def check_dia_source_radec(self, dia_sources):
  RA/DEC.
  """
  nan_mask = (dia_sources.loc[:, "ra"].isnull()
- | dia_sources.loc[:, "decl"].isnull())
+ | dia_sources.loc[:, "dec"].isnull())
  if np.any(nan_mask):
  nan_idxs = np.argwhere(nan_mask.to_numpy()).flatten()
  for nan_idx in nan_idxs:
@@ -264,7 +264,7 @@ def _radec_to_xyz(self, catalog):
  Output unit-vectors
  """
  ras = np.radians(catalog["ra"])
- decs = np.radians(catalog["decl"])
+ decs = np.radians(catalog["dec"])
  vectors = np.empty((len(ras), 3))
 
  sin_dec = np.sin(np.pi / 2 - decs)

python/lsst/ap/association/diaForcedSource.py

@@ -198,7 +198,7 @@ def _convert_from_pandas(self, input_objects):
  outputRecord.setId(obj_id)
  outputRecord.setCoord(
  geom.SpherePoint(df_row["ra"],
- df_row["decl"],
+ df_row["dec"],
  geom.degrees))
  return outputCatalog
 
@@ -236,23 +236,24 @@ def _calibrate_and_merge(self,
 
  output_catalog = diff_sources.asAstropy().to_pandas()
  output_catalog.rename(columns={"id": "diaForcedSourceId",
- "slot_PsfFlux_instFlux": "psFlux",
- "slot_PsfFlux_instFluxErr": "psFluxErr",
+ "slot_PsfFlux_instFlux": "psfFlux",
+ "slot_PsfFlux_instFluxErr": "psfFluxErr",
  "slot_Centroid_x": "x",
  "slot_Centroid_y": "y"},
  inplace=True)
- output_catalog.loc[:, "psFlux"] = diff_fluxes[:, 0]
- output_catalog.loc[:, "psFluxErr"] = diff_fluxes[:, 1]
+ output_catalog.loc[:, "psfFlux"] = diff_fluxes[:, 0]
+ output_catalog.loc[:, "psfFluxErr"] = diff_fluxes[:, 1]
 
- output_catalog["totFlux"] = direct_fluxes[:, 0]
- output_catalog["totFluxErr"] = direct_fluxes[:, 1]
+ output_catalog["scienceFlux"] = direct_fluxes[:, 0]
+ output_catalog["scienceFluxErr"] = direct_fluxes[:, 1]
 
- visit_info = direct_exp.getInfo().getVisitInfo()
+ visit_info = direct_exp.visitInfo
  ccdVisitId = direct_exp.info.id
- midPointTaiMJD = visit_info.getDate().get(system=DateTime.MJD)
+ midpointMjdTai = visit_info.date.get(system=DateTime.MJD)
  output_catalog["ccdVisitId"] = ccdVisitId
- output_catalog["midPointTai"] = midPointTaiMJD
- output_catalog["filterName"] = diff_exp.getFilter().bandLabel
+ output_catalog["midpointMjdTai"] = midpointMjdTai
+ output_catalog["band"] = diff_exp.getFilter().bandLabel
+ output_catalog["time_processed"] = DateTime.now().toPython()
 
  # Drop superfluous columns from output DataFrame.
  output_catalog.drop(columns=self.config.dropColumns, inplace=True)

python/lsst/ap/association/diaPipe.py

@@ -402,7 +402,7 @@ def run(self,
  updatedDiaObjectIds = associatedDiaSources["diaObjectId"][
  associatedDiaSources["diaObjectId"] != 0].to_numpy()
  associatedDiaSources.set_index(["diaObjectId",
- "filterName",
+ "band",
  "diaSourceId"],
  drop=False,
  inplace=True)
@@ -468,7 +468,7 @@ def run(self,
  # Store DiaSources, updated DiaObjects, and DiaForcedSources in the
  # Apdb.
  self.apdb.store(
- exposure.getInfo().getVisitInfo().getDate(),
+ exposure.visitInfo.date,
  diaCalResult.updatedDiaObjects,
  associatedDiaSources,
  diaForcedSources)
@@ -562,7 +562,7 @@ def _initialize_dia_object(self, objId):
  Id of the a nearbyObject in the Object table (`int`).
  ``nearbyObj3``
  Id of the a nearbyObject in the Object table (`int`).
- ``?PSFluxData``
+ ``?_psfFluxNdata``
  Number of data points used to calculate point source flux
  summary statistics in each bandpass (`int`).
  """
@@ -573,7 +573,7 @@ def _initialize_dia_object(self, objId):
  "nearbyObj3": 0,
  "flags": 0}
  for f in ["u", "g", "r", "i", "z", "y"]:
- new_dia_object["%sPSFluxNdata" % f] = 0
+ new_dia_object["%s_psfFluxNdata" % f] = 0
  return pd.Series(data=new_dia_object)
 
  def testDataFrameIndex(self, df):

python/lsst/ap/association/loadDiaCatalogs.py

@@ -77,13 +77,12 @@ def run(self, exposure, apdb):
  the ``diaObjectId`` column. (`pandas.DataFrame`)
  - ``diaSources`` : Complete set of DiaSources covering the input
  exposure padded by ``pixelMargin``. DataFrame is indexed by
- ``diaObjectId``, ``filterName``, ``diaSourceId`` columns.
+ ``diaObjectId``, ``band``, ``diaSourceId`` columns.
  (`pandas.DataFrame`)
  """
- visiInfo = exposure.getInfo().getVisitInfo()
  region = self._getRegion(exposure)
 
- # This is the first database query
+ # This is the first database query.
  try:
  diaObjects = self.loadDiaObjects(region, apdb)
  except (OperationalError, ProgrammingError) as e:
@@ -92,17 +91,11 @@ def run(self, exposure, apdb):
  "make_apdb.py first? If you did, some other error occurred "
  "during database access of the DiaObject table.") from e
 
- dateTime = visiInfo.getDate()
+ dateTime = exposure.visitInfo.date
 
- diaSources = self.loadDiaSources(diaObjects,
- region,
- dateTime,
- apdb)
+ diaSources = self.loadDiaSources(diaObjects, region, dateTime, apdb)
 
- diaForcedSources = self.loadDiaForcedSources(diaObjects,
- region,
- dateTime,
- apdb)
+ diaForcedSources = self.loadDiaForcedSources(diaObjects, region, dateTime, apdb)
 
  return pipeBase.Struct(
  diaObjects=diaObjects,
@@ -172,12 +165,12 @@ def loadDiaSources(self, diaObjects, region, dateTime, apdb):
  # If no area is specified return an empty DataFrame with the
  # the column used for indexing later in AssociationTask.
  diaSources = pd.DataFrame(columns=["diaObjectId",
- "filterName",
+ "band",
  "diaSourceId"])
  else:
  diaSources = apdb.getDiaSources(region, diaObjects.loc[:, "diaObjectId"], dateTime)
 
- diaSources.set_index(["diaObjectId", "filterName", "diaSourceId"],
+ diaSources.set_index(["diaObjectId", "band", "diaSourceId"],
  drop=False,
  inplace=True)
  if diaSources.index.has_duplicates:
@@ -187,7 +180,7 @@ def loadDiaSources(self, diaObjects, region, dateTime, apdb):
  # Drop duplicates via index and keep the first appearance. Reset
  # due to the index shape being slight different thatn expected.
  diaSources = diaSources.groupby(diaSources.index).first().reset_index(drop=True)
- diaSources.set_index(["diaObjectId", "filterName", "diaSourceId"],
+ diaSources.set_index(["diaObjectId", "band", "diaSourceId"],
  drop=False,
  inplace=True)
 
@@ -233,7 +226,7 @@ def loadDiaForcedSources(self, diaObjects, region, dateTime, apdb):
  "Duplicate DiaForcedSources loaded from the Apdb. This may "
  "cause downstream pipeline issues. Dropping duplicated rows.")
  # Drop duplicates via index and keep the first appearance. Reset
- # due to the index shape being slight different thatn expected.
+ # due to the index shape being slightly different than expected.
  diaForcedSources = diaForcedSources.groupby(diaForcedSources.index).first()
  diaForcedSources.reset_index(drop=True, inplace=True)
  diaForcedSources.set_index(["diaObjectId", "diaForcedSourceId"],

python/lsst/ap/association/packageAlerts.py

@@ -96,14 +96,14 @@ def run(self,
  ----------
  diaSourceCat : `pandas.DataFrame`
  New DiaSources to package. DataFrame should be indexed on
- ``["diaObjectId", "filterName", "diaSourceId"]``
+ ``["diaObjectId", "band", "diaSourceId"]``
  diaObjectCat : `pandas.DataFrame`
  New and updated DiaObjects matched to the new DiaSources. DataFrame
  is indexed on ``["diaObjectId"]``
  diaSrcHistory : `pandas.DataFrame`
  12 month history of DiaSources matched to the DiaObjects. Excludes
  the newest DiaSource and is indexed on
- ``["diaObjectId", "filterName", "diaSourceId"]``
+ ``["diaObjectId", "band", "diaSourceId"]``
  diaForcedSources : `pandas.DataFrame`
  12 month history of DiaForcedSources matched to the DiaObjects.
  ``["diaObjectId"]``
@@ -138,7 +138,7 @@ def run(self,
  objSourceHistory = None
  objDiaForcedSources = diaForcedSources.loc[srcIndex[0]]
  sphPoint = geom.SpherePoint(diaSource["ra"],
- diaSource["decl"],
+ diaSource["dec"],
  geom.degrees)
 
  cutoutExtent = self.createDiaSourceExtent(diaSource["bboxSize"])