Skip to content

Commit

Permalink
Make trailedAssociatorTask
Browse files Browse the repository at this point in the history
Make trailedAssociatorTask which filters out trails whose lengths are above 0.416 arcseconds/second in length.
Updated unit test and added test_trailedSourceFilter.py. Refactored doLongTrailFilter to doTrailedSourceFilter.
  • Loading branch information
bsmartradio committed Aug 21, 2023
1 parent 6758148 commit d7c4173
Show file tree
Hide file tree
Showing 7 changed files with 298 additions and 15 deletions.
1 change: 1 addition & 0 deletions python/lsst/ap/association/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.

from .version import *
from .trailedSourceFilter import *
from .association import *
from .diaForcedSource import *
from .loadDiaCatalogs import *
Expand Down
32 changes: 30 additions & 2 deletions python/lsst/ap/association/association.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import lsst.pex.config as pexConfig
import lsst.pipe.base as pipeBase
from lsst.utils.timer import timeMethod
from .trailedSourceFilter import TrailedSourceFilterTask

# Enforce an error for unsafe column/array value setting in pandas.
pd.options.mode.chained_assignment = 'raise'
Expand All @@ -47,6 +48,18 @@ class AssociationConfig(pexConfig.Config):
default=1.0,
)

trailedSourceFilter = pexConfig.ConfigurableField(
target=TrailedSourceFilterTask,
doc="Subtask to filter artifact candidates based on morphological "
"criteria, i.g. those that appear to be streaks.",
)

doTrailedSourceFilter = pexConfig.Field(
doc="Set flag for trailed source filter subtask to run.",
dtype=bool,
default=True,
)


class AssociationTask(pipeBase.Task):
"""Associate DIAOSources into existing DIAObjects.
Expand All @@ -60,10 +73,16 @@ class AssociationTask(pipeBase.Task):
ConfigClass = AssociationConfig
_DefaultName = "association"

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if self.config.doTrailedSourceFilter:
self.makeSubtask("trailedSourceFilter")

@timeMethod
def run(self,
diaSources,
diaObjects):
diaObjects,
exposure=None):
"""Associate the new DiaSources with existing DiaObjects.
Parameters
Expand All @@ -72,6 +91,9 @@ def run(self,
New DIASources to be associated with existing DIAObjects.
diaObjects : `pandas.DataFrame`
Existing diaObjects from the Apdb.
exposure : `pandas.DataFrame` optional
Calibrated exposure differenced with a template image during
image differencing.
Returns
-------
Expand All @@ -98,7 +120,13 @@ def run(self,
nUpdatedDiaObjects=0,
nUnassociatedDiaObjects=0)

matchResult = self.associate_sources(diaObjects, diaSources)
if self.config.doTrailedSourceFilter:
diaTrailedResult = self.trailedSourceFilter.run(diaSources, exposure)

matchResult = self.associate_sources(diaObjects, diaTrailedResult.diaSources)

else:
matchResult = self.associate_sources(diaObjects, diaSources)

mask = matchResult.diaSources["diaObjectId"] != 0

Expand Down
17 changes: 11 additions & 6 deletions python/lsst/ap/association/diaPipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@
Currently loads directly from the Apdb rather than pre-loading.
"""

__all__ = ("DiaPipelineConfig",
"DiaPipelineTask",
"DiaPipelineConnections")

import pandas as pd

import lsst.dax.apdb as daxApdb
Expand All @@ -41,13 +45,10 @@
AssociationTask,
DiaForcedSourceTask,
LoadDiaCatalogsTask,
PackageAlertsTask)
PackageAlertsTask,
TrailedSourceFilterTask)
from lsst.ap.association.ssoAssociation import SolarSystemAssociationTask

__all__ = ("DiaPipelineConfig",
"DiaPipelineTask",
"DiaPipelineConnections")


class DiaPipelineConnections(
pipeBase.PipelineTaskConnections,
Expand Down Expand Up @@ -221,6 +222,10 @@ class DiaPipelineConfig(pipeBase.PipelineTaskConfig,
target=AssociationTask,
doc="Task used to associate DiaSources with DiaObjects.",
)
trailedFilter = pexConfig.ConfigurableField(
target=TrailedSourceFilterTask,
doc="Task used to find trailed DiaSources.",
)
doSolarSystemAssociation = pexConfig.Field(
dtype=bool,
default=False,
Expand Down Expand Up @@ -368,7 +373,7 @@ def run(self,

# Associate new DiaSources with existing DiaObjects.
assocResults = self.associator.run(diaSourceTable,
loaderResult.diaObjects)
loaderResult.diaObjects, exposure=exposure)
if self.config.doSolarSystemAssociation:
ssoAssocResult = self.solarSystemAssociator.run(
assocResults.unAssocDiaSources,
Expand Down
114 changes: 114 additions & 0 deletions python/lsst/ap/association/trailedSourceFilter.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
# This file is part of ap_association.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (https://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

"""A simple implementation of trailed source filtering for the association
task.
"""

__all__ = ["TrailedSourceFilterTask", "TrailedSourceFilterConfig"]

import lsst.pex.config as pexConfig
import lsst.pipe.base as pipeBase
from lsst.utils.timer import timeMethod

# import numpy as np
import pandas as pd

# Enforce an error for unsafe column/array value setting in pandas.
pd.options.mode.chained_assignment = 'raise'


class TrailedSourceFilterConfig(pexConfig.Config):
"""Config class for TrailedSourceFilterTask.
"""
maxTrailLength = pexConfig.Field(
dtype=float,
doc='Maximum trail length permitted is less than 10 degrees/day. '
'This is a rate of 0.416 arcseconds per second. As trail length'
' is measured in arcseconds, it is dependant on the length of '
'the exposure.',
default=0.416,
)


class TrailedSourceFilterTask(pipeBase.Task):
"""Find trailed sources in DIAObjects.
"""

ConfigClass = TrailedSourceFilterConfig
_DefaultName = "trailedAssociation"

@timeMethod
def run(self,
dia_sources, exposure):
"""Find trailed sources which have not been filtered out and will
not be included in the diaSource catalog.
Parameters
----------
dia_sources : `pandas.DataFrame`
New DIASources to be checked for trailed sources.
exposure : `pandas.DataFrame`
Calibrated exposure differenced with a template image during
image differencing.
Returns
-------
result : `lsst.pipe.base.Struct`
Results struct with components.
- ``"dia_sources"`` : DiaSource table that is free from unwanted
trailed sources (`pandas.DataFrame`)
- ``"trailed_dia_sources"`` : DiaSources that have trailed more
than 0.416 arcseconds/second*exposure_time(`pandas.DataFrame`)
"""
trail_mask = self.check_dia_source_trail(dia_sources, exposure)

return pipeBase.Struct(
diaSources=dia_sources[~trail_mask].reset_index(drop=True),
trailedDiaSources=dia_sources[trail_mask].reset_index(drop=True))

def check_dia_source_trail(self, dia_sources, exposure):
"""Check that all DiaSources have trails.
Creates a mask for sources with lengths greater than 0.416
arcseconds/second times the exposure time.
Parameters
----------
dia_sources : `pandas.DataFrame`
Input DiaSources to check for trail lengths.
exposure : `pandas.DataFrame`
Calibrated exposure differenced with a template image during
image differencing.
Returns
-------
trail_mask : `pandas.DataFrame`
Boolean mask for dia_sources which are greater than the
cuttoff length.
"""
exposure_time = exposure.getInfo().getVisitInfo().getExposureTime()
trail_mask = (dia_sources.loc[:, "trailLength"].values[:] >= self.config.maxTrailLength*exposure_time)

return trail_mask
54 changes: 48 additions & 6 deletions tests/test_association_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
import numpy as np
import pandas as pd
import unittest

import lsst.afw.image as afwImage
from lsst.afw.coord import Weather
import lsst.geom as geom
import lsst.utils.tests

Expand All @@ -46,20 +47,38 @@ def setUp(self):
self.diaSources = pd.DataFrame(data=[
{"ra": 0.04*idx + scatter*rng.uniform(-1, 1),
"dec": 0.04*idx + scatter*rng.uniform(-1, 1),
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0}
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0, "trailLength": 5.5*idx}
for idx in range(self.nSources)])
self.diaSourceZeroScatter = pd.DataFrame(data=[
{"ra": 0.04*idx,
"dec": 0.04*idx,
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0}
"diaSourceId": idx + 1 + self.nObjects, "diaObjectId": 0, "trailLength": 5.5*idx}
for idx in range(self.nSources)])
exposureId = 5
exposureTime = 30
boresightRotAngle = 45.6 * lsst.geom.degrees
weather = Weather(1.1, 2.2, 0.3)
visitInfo = afwImage.VisitInfo(
exposureId=exposureId,
exposureTime=exposureTime,
boresightRotAngle=boresightRotAngle,
weather=weather,
)
exposureInfo = afwImage.ExposureInfo()
exposureInfo.setVisitInfo(visitInfo)
maskedImage = afwImage.MaskedImageF(lsst.geom.Extent2I(64, 64))
self.exposure = afwImage.ExposureF(maskedImage, exposureInfo)

def test_run(self):
"""Test the full task by associating a set of diaSources to
existing diaObjects.
"""
assocTask = AssociationTask()
results = assocTask.run(self.diaSources, self.diaObjects)

config = AssociationTask.ConfigClass()
config.doTrailedSourceFilter = False
assocTask = AssociationTask(config=config)

results = assocTask.run(self.diaSources, self.diaObjects, self.exposure)

self.assertEqual(results.nUpdatedDiaObjects, len(self.diaObjects) - 1)
self.assertEqual(results.nUnassociatedDiaObjects, 1)
Expand All @@ -75,13 +94,36 @@ def test_run(self):
[0]):
self.assertEqual(test_obj_id, expected_obj_id)

def test_run_trailed_sources(self):
"""Test the full task by associating a set of diaSources to
existing diaObjects when trailed sources are filtered.
"""

assocTask = AssociationTask()

results = assocTask.run(self.diaSources, self.diaObjects, self.exposure)

self.assertEqual(results.nUpdatedDiaObjects, len(self.diaObjects) - 3)
self.assertEqual(results.nUnassociatedDiaObjects, 3)
self.assertEqual(len(results.matchedDiaSources),
len(self.diaObjects) - 3)
self.assertEqual(len(results.unAssocDiaSources), 1)
for test_obj_id, expected_obj_id in zip(
results.matchedDiaSources["diaObjectId"].to_numpy(),
[1, 2, 3, 4]):
self.assertEqual(test_obj_id, expected_obj_id)
for test_obj_id, expected_obj_id in zip(
results.unAssocDiaSources["diaObjectId"].to_numpy(),
[0]):
self.assertEqual(test_obj_id, expected_obj_id)

def test_run_no_existing_objects(self):
"""Test the run method with a completely empty database.
"""
assocTask = AssociationTask()
results = assocTask.run(
self.diaSources,
pd.DataFrame(columns=["ra", "dec", "diaObjectId"]))
pd.DataFrame(columns=["ra", "dec", "diaObjectId", "trailLength"]), self.exposure)
self.assertEqual(results.nUpdatedDiaObjects, 0)
self.assertEqual(results.nUnassociatedDiaObjects, 0)
self.assertEqual(len(results.matchedDiaSources), 0)
Expand Down
2 changes: 1 addition & 1 deletion tests/test_diaPipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def solarSystemAssociator_run(self, unAssocDiaSources, solarSystemObjectTable, d
unAssocDiaSources=MagicMock(spec=pd.DataFrame()))

@lsst.utils.timer.timeMethod
def associator_run(self, table, diaObjects):
def associator_run(self, table, diaObjects, exposure):
return lsst.pipe.base.Struct(nUpdatedDiaObjects=2, nUnassociatedDiaObjects=3,
matchedDiaSources=MagicMock(spec=pd.DataFrame()),
unAssocDiaSources=MagicMock(spec=pd.DataFrame()))
Expand Down
Loading

0 comments on commit d7c4173

Please sign in to comment.