Source Importer Filter et al.

db/2026-04-30_spectra_ishost.sql

@@ -0,0 +1,7 @@
+ALTER TABLE wantedspectra ADD COLUMN is_host boolean;
+ALTER TABLE wantedspectra ADD COLUMN ra double precision;
+ALTER TABLE wantedspectra ADD COLUMN dec double precision;
+CREATE INDEX ix_wantedspectra_q3c ON wantedspectra( q3c_ang2ipix( ra, dec ) );
+ALTER TABLE plannedspectra ADD COLUMN is_host boolean;
+ALTER TABLE plannedspectra ADD COLUMN wantspec_id text;
+ALTER TABLE spectruminfo ALTER COLUMN is_host DROP NOT NULL;

docker-compose.yaml

@@ -3,7 +3,7 @@
 
 services:
   kafka-server:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-kafka-test:${DOCKER_VERSION-test20260225}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-kafka-test:${DOCKER_VERSION-test20260428}
     build:
       context: ./docker/kafka
     environment:
@@ -32,7 +32,7 @@ services:
       retries: 10
 
   postgres:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-postgres:${DOCKER_VERSION:-test20260225}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-postgres:${DOCKER_VERSION:-test20260428}
     build:
       context: ./docker/postgres
       target: postgres
@@ -53,7 +53,7 @@ services:
       retries: 10
 
   mongodb:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-mongodb:${DOCKER_VERSION:-test20260225}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-mongodb:${DOCKER_VERSION:-test20260428}
     build:
       context: ./docker/mongodb
     environment:
@@ -80,7 +80,7 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260225}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260428}
     build:
       context: ./docker/webserver
       target: shell
@@ -101,7 +101,7 @@ services:
     depends_on:
       createdb:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-query-runner:${DOCKER_VERSION:-test20260225}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-query-runner:${DOCKER_VERSION:-test20260428}
     build:
       context: ./docker/query_runner
       target: queryrunner
@@ -132,7 +132,7 @@ services:
         condition: service_started
       queryrunner:
         condition: service_started
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-webap:${DOCKER_VERSION:-test20260225}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-webap:${DOCKER_VERSION:-test20260428}
     build:
       context: ./docker/webserver
       target: webserver
@@ -188,7 +188,7 @@ services:
         condition: service_healthy
       createdb:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260225}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260428}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert
@@ -236,7 +236,7 @@ services:
         condition: service_healthy
       mongodb:
         condition: service_healthy
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260225}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260428}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert
@@ -279,7 +279,7 @@ services:
     working_dir: /code
 
   makeinstall:
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260225}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260428}
     user: ${USERID:-0}:${GROUPID:-0}
     volumes:
       - type: bind
@@ -314,7 +314,7 @@ services:
         condition: service_started   # TODO : health test for webap
       makeinstall:
         condition: service_completed_successfully
-    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260225}
+    image: ${DOCKER_ARCHIVE:-ghcr.io/lsstdesc}/fastdb-shell:${DOCKER_VERSION:-test20260428}
     environment:
       - MONGODB_HOST=mongodb
       - MONGODB_DBNAME=brokeralert

docker/webserver/Dockerfile

@@ -42,22 +42,28 @@ RUN curl -fsSL https://www.mongodb.org/static/pgp/server-8.0.asc | \
    gpg -o /usr/share/keyrings/mongodb-server-8.0.gpg \
    --dearmor
 
-RUN echo "deb [ signed-by=/usr/share/keyrings/mongodb-server-8.0.gpg ] http://repo.mongodb.org/apt/debian bookworm/mongodb-org/8.0 main" | tee /etc/apt/sources.list.d/mongodb-org-8.0.list
+RUN echo "deb [ signed-by=/usr/share/keyrings/mongodb-server-8.0.gpg ] http://repo.mongodb.org/apt/debian trixie/mongodb-org/8.0 main" | tee /etc/apt/sources.list.d/mongodb-org-8.0.list
 
 RUN apt-get update \
   && DEBIAN_FRONTEND="noninteractive" TZ="UTC" apt-get -y install -y --no-install-recommends \
-          mongodb-mongosh mongodb-database-tools \
-  && apt-get clean \
+          mongodb-mongosh \
+ && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
+# ...mongodb-database-tools isn't always in the archive I just added!  It's missing for
+#   trixie, and even for bookworm it's missing for arm64.  This mongo
+#   archive is kind of a disaster.  Installation of mongodump and mongorestore has been moved to build.
+
 
 # ======================================================================
 FROM base AS build
 
+WORKDIR /usr/src
+
 RUN DEBIAN_FRONTEND="noninteractive" TZ="UTC" \
     apt-get update \
     && DEBIAN_FRONTEND="noninteractive" TZ="UTC" \
-    apt-get -y install -y python3-pip python3-venv git libpq-dev
+    apt-get -y install -y python3-pip python3-venv git libpq-dev golang
 
 RUN mkdir /venv
 RUN python3 -mvenv /venv
@@ -94,18 +100,19 @@ RUN source /venv/bin/activate && \
 COPY pubsub.patch /tmp/pubsub.patch
 RUN patch /venv/lib/python3.13/site-packages/pittgoogle/pubsub.py /tmp/pubsub.patch
 
-# We have to install fink-client manually because right now it pins its
-#  requirement to fastavro 1.9.4, but I have failed to pip install
-#  fastavro 1.9.4 (is it a python 3.13 thing?)
-
-# WORKDIR /usr/src
-# RUN git clone https://github.com/astrolabsoftware/fink-client.git
-# RUN cat fink-client/requirements.txt | perl -pe 's/fastavro==1.9.4/fastavro>=1.9.4/' > fink-client/requirements-new.txt
-# RUN mv fink-client/requirements-new.txt fink-client/requirements.txt
-# RUN cd fink-client \
-#     && source /venv/bin/activate \
-#     && pip install .
-
+# Irritating that the mongodb debian apt aarchive doesn't have debian
+#   arm packages for mongodb-database-tools, nor does it have them for
+#   any architecture for trixie.  So, build from source.
+#
+# (...and even though this needs golang installed for ./make build to even run,
+#  the first thing it does it download go...)
+RUN git clone https://github.com/mongodb/mongo-tools.git \
+  && cd mongo-tools \
+  && ./make build \
+  && cp -p bin/mongodump /usr/local/bin \
+  && cp -p bin/mongorestore /usr/local/bin \
+  && cd .. \
+  && rm -rf mongo-tools
 
 # ======================================================================
 # This is for the test webserver.  It installs crappy keys so you
@@ -115,6 +122,8 @@ RUN patch /venv/lib/python3.13/site-packages/pittgoogle/pubsub.py /tmp/pubsub.pa
 FROM base AS test
 
 COPY --from=build /venv/ /venv/
+COPY --from=build /usr/local/bin/mongodump /usr/local/bin/mongodump
+COPY --from=build /usr/local/bin/mongorestore /usr/local/bin/mongorestore
 ENV PATH=/venv/bin:$PATH
 
 RUN mkdir -p /fastdb/webserver
@@ -137,6 +146,8 @@ ENTRYPOINT [ "gunicorn", "--certfile", "/usr/src/cert.pem", "--keyfile", "/usr/s
 FROM base AS shell
 
 COPY --from=build /venv/ /venv/
+COPY --from=build /usr/local/bin/mongodump /usr/local/bin/mongodump
+COPY --from=build /usr/local/bin/mongorestore /usr/local/bin/mongorestore
 ENV PATH=/venv/bin:$PATH
 
 RUN mkdir -p /fastdb/webserver
@@ -150,6 +161,8 @@ ENTRYPOINT [ "tail", "-f", "/etc/issue" ]
 FROM base AS webserver
 
 COPY --from=build /venv/ /venv/
+COPY --from=build /usr/local/bin/mongodump /usr/local/bin/mongodump
+COPY --from=build /usr/local/bin/mongorestore /usr/local/bin/mongorestore
 ENV PATH=/venv/bin:$PATH
 
 RUN mkdir -p /fastdb/webserver

src/db.py

@@ -452,8 +452,8 @@ def construct_pgsql_where_clause( searchspec, where="WHERE", **kwargs ):
                 where = " AND"
             else:
                 q += sql.SQL( "{where} {field} LIKE %({sfield}_contains)s" ).format( where=sql.SQL(where),
-                                                                                           field=sql.Identifier(field),
-                                                                                           sfield=sql.SQL(field) )
+                                                                                     field=sql.Identifier(field),
+                                                                                     sfield=sql.SQL(field) )
                 subdict[f'{field}_contains'] = f"%{kwargs[f'{field}_contains']}%"
                 where = " AND"
             del kwargs[f'{field}_contains']
@@ -466,7 +466,8 @@ def construct_pgsql_where_clause( searchspec, where="WHERE", **kwargs ):
             q += sql.SQL( "{where} {field}>=%({sfield}_min)s" ).format( where=sql.SQL(where),
                                                                         field=sql.Identifier(field),
                                                                         sfield=sql.SQL(field) )
-            subdict['f{field}_min'] = kwargs[f'{field}_min']
+            subdict[f'{field}_min'] = kwargs[f'{field}_min']
+            where = " AND"
             del kwargs[f'{field}_min']
 
         if f'{field}_max' in kwargs:
@@ -475,12 +476,13 @@ def construct_pgsql_where_clause( searchspec, where="WHERE", **kwargs ):
             if util.isSequence( f'{field}_max' ):
                 raise ValueError( f"{field}_max can't be a list" )
             q += sql.SQL( "{where} {field}<=%({sfield}_max)s" ).format( where=sql.SQL(where),
-                                                                        field=sql.Identifier(field) )
+                                                                        field=sql.Identifier(field),
+                                                                        sfield=sql.SQL(field) )
             subdict[f'{field}_max'] = kwargs[f'{field}_max']
             where = " AND"
             del kwargs[f'{field}_max']
 
-    return q, subdict, set(kwargs.keys())
+    return q, subdict, set(kwargs.keys()), where
 
 
 # ======================================================================

src/ltcv.py

@@ -2106,7 +2106,7 @@ def get_hot_ltcvs( processing_version, position_processing_version=None,
 
       Returns
       -------
-        ltcvdf, objinfo, hostdf
+        ltcvdf, objinfo
 
         ltcvdf: pandas.DataFrame
            A dataframe with lightcurves. It is sorted and indexed by
@@ -2173,7 +2173,7 @@ def get_hot_ltcvs( processing_version, position_processing_version=None,
            should never have fewer rows tan ltcvdf.)
 
         hostdf: None
-           NOT CURRENTLY SUPPORTED.
+           NOT CURRENTLY SUPPORTED.... so not returned
 
         *A note on diaobjectid of sources : LSST may put more than one
          diaObjectId at the same point in the sky.  What's more,

src/services/brokerconsumer.py

@@ -474,6 +474,10 @@ def _filter_dict_to_table( cls, alertdict, tablemeta ):
 
     @classmethod
     def _wrangle_object( cls, msg, metamsg ):
+        # Throw out things with diaObjectId 0; those are bad
+        if msg['diaSource']['diaObjectId'] in [0, None]:
+            return None
+
         obj = { 'diaobjectid': msg['diaSource']['diaObjectId'],
                 'savetime': metamsg['savetime'],
                 'diaobjectposition': None }
@@ -530,13 +534,14 @@ def _wrangle_diaforcedsource_extra( cls, submsg, metamsg, msg ):
     def _wrangle_all_standard_lsst_fields( self, metamsg, msg ):
         obj = self._wrangle_object( msg, metamsg )
         # Basic sanity check
-        try:
-            np.int64( obj['diaobjectid'] )
-        except Exception as ex:
-            self.countlogger.error( f"Got an alert with diaSource.diaObjectId={obj['diaobjectid']} "
-                                    f"(type {type(obj['diaobjectid'])}), which isn't "
-                                    f"a 64-bit integer.  Skipping this alert!  Exception: {ex}" )
-            return None
+        if obj is not None:
+            try:
+                np.int64( obj['diaobjectid'] )
+            except Exception as ex:
+                self.countlogger.error( f"Got an alert with diaSource.diaObjectId={obj['diaobjectid']} "
+                                        f"(type {type(obj['diaobjectid'])}), which isn't "
+                                        f"a 64-bit integer.  Skipping this alert!  Exception: {ex}" )
+                return None
 
         # TODO : more sanity checks.
 
@@ -559,6 +564,7 @@ def _wrangle_all_standard_lsst_fields( self, metamsg, msg ):
 
         if any( ( f in msg and f is not None ) for f in [ 'cutoutDifference', 'cutoutScience', 'cutoutTemplate' ] ):
             thumbnails = { 'diasourceid': msg['diaSource']['diaSourceId'],
+                           'diaobjectid': msg['diaSource']['diaObjectId'],
                            'savetime': metamsg['savetime'] }
             thumbnails.update( { f.lower(): msg[f] if f in msg else None
                                  for f in ['cutoutDifference', 'cutoutScience', 'cutoutTemplate' ] } )